svn commit: r1737721 - /uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java

[challngr]

Author: challngr
Date: Mon Apr  4 18:33:54 2016
New Revision: 1737721

URL: http://svn.apache.org/viewvc?rev=1737721=rev
Log:
UIMA-4882 Extend init_wait semantics to include "no recorded job time yet."

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java?rev=1737721=1737720=1737721=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
 Mon Apr  4 18:33:54 2016
@@ -569,7 +569,7 @@ public class RmJob
 sharesByMachine.clear();
 }
 
-/**
+   /**
  * I've shrunk or this share has nothing left to do.  Remove this specific 
share.
  */
 public void removeShare(Share share)
@@ -1000,7 +1000,25 @@ public class RmJob
 {
String methodName = "getPrjCap";  // want this to 
line up with getJobCap in logs
 
-if ( init_wait || Double.isNaN(time_per_item) || (time_per_item == 
0.0)) {   // no cap if not initialized, or no per-itme time yet
+// UIMA-4882 jrc
+// Must enhance semantics of init_wait to mean "nothing initialized, 
and have never seen any
+// execution time for the job."  This accounts for the moment after a 
job initializes, and before it
+// gets anything running and helps to throttle expansion until a job 
starts to run.
+//
+// After initialization, the time_per_item will be quite small but 
non-zero, so we'll tend to predict
+// a future cap as the moral equicalent of "not too many more needed". 
 For installations without
+// doubling, or where doubling is too fast, this leads to better 
controlled expansion if the job
+// actually is going to compete soon.  
+//
+// The other part of this update includes the OR updating its 
"time_per_item" to account for
+// work items in progress as well as work items completed, so we're 
guarantteed to get a
+// time_per_item != 0 shortly after first initialization.
+//
+// (We update init_wait here because it's used later and needs to be 
used with the same 
+//  semantics as is used here.)
+init_wait = init_wait || Double.isNaN(time_per_item) || (time_per_item 
== 0.0);
+
+if ( init_wait ) {   // no cap if not initialized, or no per-itme time 
yet
 logger.info(methodName, getId(), username, "Cannot predict cap: 
init_wait", init_wait, "|| time_per_item", time_per_item);
 return Integer.MAX_VALUE;
 }

svn commit: r1730949 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database: DbUtil.java StateServicesDb.java

[challngr]

Author: challngr
Date: Wed Feb 17 21:37:22 2016
New Revision: 1730949

URL: http://svn.apache.org/viewvc?rev=1730949=rev
Log:
UIMA-4577 Correctly handle keys in property files to avoid malformed CQL.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java?rev=1730949=1730948=1730949=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java
 Wed Feb 17 21:37:22 2016
@@ -205,6 +205,9 @@ class DbUtil
 
 }
 
+/**
+ * Imporant: do not pass key fields in the props or this will barf. 
+ */
 static String mkUpdate(String table, String key, Object... props)
 {
 int len = props.length;
@@ -212,20 +215,21 @@ class DbUtil
 buf.append(table);
 buf.append(" SET ");
 
+// NOTE: The property set must NOT contain any key fields or this is 
likely to barf.  Caller
+//   must insure.
 for ( int i = 0; i < len; i+=2) {
 IDbProperty prop = (IDbProperty) props[i];
-if ( !prop.isPrimaryKey()) {  // not allowed to 
update this
-  // we allow it in 
'props' so callers can
-  // simply call 
update and expect the right
-  // thing to happen
 
-buf.append(prop.columnName());
-buf.append("=");
-buf.append(rep(prop, props[i+1]));
-if ( i + 2 < len ) {
-buf.append(",");
-}  
+if ( prop.isPrimaryKey() ) {
+throw new IllegalArgumentException("Primary key not allowed in 
UPDATE");
 }
+
+buf.append(prop.columnName());
+buf.append("=");
+buf.append(rep(prop, props[i+1]));
+if ( i + 2 < len ) {
+buf.append(",");
+}  
 }
 buf.append(" WHERE ");
 buf.append(key);

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java?rev=1730949=1730948=1730949=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
 Wed Feb 17 21:37:22 2016
@@ -349,17 +349,18 @@ public class StateServicesDb
 try {
 h = dbManager.open();
 Map<IDbProperty, Object> map = mkMap(serviceId, table, converter, 
props);
-Object[] updates = new Object[props.size()*2];
+List tmp = new ArrayList();
 int i = 0;
 for ( IDbProperty k : map.keySet() ) {
 if ( logger.isTrace() ) {
 logger.trace(methodName, null, "Updating", k.columnName(), 
"with", map.get(k));
 }
-updates[i++] = k;
-updates[i++] = map.get(k);
+if ( k.isPrimaryKey() ) continue;// we do not get to 
update this
+tmp.add(k);
+tmp.add(map.get(k));
 }
 
-h.updateProperties(table, key, updates);
+h.updateProperties(table, key, tmp.toArray(new 
Object[tmp.size()]));
 return true;
 } catch ( Exception e ) {
 logger.error(methodName, null, "Unable to update properties for 
service", key, "table", table, ":", e);

svn commit: r1730702 - /uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy

[challngr]

Author: challngr
Date: Tue Feb 16 15:48:22 2016
New Revision: 1730702

URL: http://svn.apache.org/viewvc?rev=1730702=rev
Log:
UIMA-4577 Eliminate color prefix-chars doing cqlsh in rm_qoccupancy.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy?rev=1730702=1730701=1730702=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy Tue Feb 16 
15:48:22 2016
@@ -80,10 +80,6 @@ class DuccRmQOccupancy(DuccUtil):
 shares = {}
 header = []
 for l in lines:
-# First character of first line may be 0x1B ESC
-if len(l) > 0 and not l[0] in string.printable:
-print 'Ignoring line', '[' + l + '] hex:', binascii.hexlify(l)
-continue
 if ( l == '' ):
 continue
 if ( '---' in l ):
@@ -124,6 +120,7 @@ class DuccRmQOccupancy(DuccUtil):
 DH = self.DUCC_HOME
 dbn = self.ducc_properties.get('ducc.database.host')
 
+os.environ['TERM'] = 'dumb'  # insure no colors.  --no-color isn't 
inhibiting colors in this shell for some reason.
 CMD = [DH + '/cassandra-server/bin/cqlsh', dbn, '-u', 'guest', '-p', 
'guest', '-e', '"select * from ducc.rmnodes; select * from ducc.rmshares;"']
 CMD = ' '.join(CMD)
 
@@ -131,7 +128,7 @@ class DuccRmQOccupancy(DuccUtil):
 proc = subprocess.Popen(CMD, bufsize=0, stdout=subprocess.PIPE, 
shell=True)
 for line in proc.stdout:
 # print line.strip()
-lines.append(line)
+lines.append(line.strip())
 
 nodes, shares = self.rmnodes(lines)
 self.format(nodes, shares)

svn commit: r1727979 [2/3] - in /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook: ./ images/ducc-internals/ part5/

[challngr]

Added: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-rm.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-rm.tex?rev=1727979=auto
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-rm.tex
 (added)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-rm.tex
 Mon Feb  1 17:36:08 2016
@@ -0,0 +1,1109 @@
+% 
+% Licensed to the Apache Software Foundation (ASF) under one
+% or more contributor license agreements.  See the NOTICE file
+% distributed with this work for additional information
+% regarding copyright ownership.  The ASF licenses this file
+% to you under the Apache License, Version 2.0 (the
+% "License"); you may not use this file except in compliance
+% with the License.  You may obtain a copy of the License at
+% 
+%   http://www.apache.org/licenses/LICENSE-2.0
+% 
+% Unless required by applicable law or agreed to in writing,
+% software distributed under the License is distributed on an
+% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+% KIND, either express or implied.  See the License for the
+% specific language governing permissions and limitations
+% under the License.
+% 
+
+
+This chapter provides architectural and implementation details for the DUCC
+Resource Manager, referred to as the ``RM''.
+\section{Introduction}
+
+The DUCC Resource Manager is responsible for apportioning cluster resources to 
the
+collection of ``work'' to be run by users.  Work is classified into several 
categories.  As
+exposed in the public interface, these categories are:
+
+\begin{description}
+  \item[Fair-Share Job] This is a UIMA/AS job, consisting of a minimum of two 
processes and 
+a potential maximum of as many processes as physically fit on a cluster.  
The work
+executed by the processes is parallel, enabling the RM to expand or 
contract
+the job by allocating or deallocating processes as needed to balance the 
load.
+
+Load is balanced using a weighted fair-share policy in which all users are 
apportioned an
+equitable amount of the cluster resources, where ``cluster resources'' 
refers only to real
+memory on the cluster nodes.
+
+  \item[Service Instance] This is any arbitrary process that DUCC manages as a 
``service''.
+Services are registered with the Service Manager and may be comprised of 
multiple physical
+processes.  (See the DuccBook for details of DUCC Service Management.)  
The RM schedules these
+processes as singletons, using a non-preemptive policy (FIXED\_SHARE or 
RESERVE).
+
+  \item[Arbitrary Process or ``Managed Reservation''] These are singleton 
processes of any type, scheduled
+using a FIXED\_SHARE policy.
+
+  \item[Fixed-Share Job] This is a UIMA-AS job scheduled with a 
non-preemptable, i.e. FIXED\_SHARE
+policy.
+
+  \item[Reservation] This is a request for a dedicated full machine.
+\end{description}
+
+The RM is a memory scheduler only.  The use case which justifies DUCC is 
UIMA-AS jobs, each of
+which consists a variable number of parallel processes, each of which requires 
large amounts of memory, usually
+on the order of 16GB or more.  Memory requirements completely overwhelm other 
resource
+requirements, so that jobs scheduled by their declared memory sizes usually 
get sufficient
+other resource such as CPU. 
+
+\section{Vocabulary}
+In order to understand RM it is necessary to understand some of the 
language used in RM.
+
+\begin{description}
+\item[quantum] This is the smallest memory size of an allocation, defined 
in multiples of GB.  It
+  is defined globally in {\em ducc.properties} and may be overridden in 
{\em ducc.classes} for
+  top-level nodepools.  See the DuccBook for more details.
+  
+  Note that although DUCC defines a quantum, most of the RM does not
+  use quantum at all; instead it generalizes quantum into {\em qshare}, 
{\em nshare},
+  and {\em order} as defined below.  When a schedule is returned to the 
Orchestrator, the
+  allocations, in terms of quanta, are translated back to memory 
allocations using this
+  configured {\em quantum}.
+  
+\item[qshare] This is an abstract memory allocation representing exactly 
one {\em quantum}.  Memory
+  allocations are made in terms of some multiple of {\em qshares}.
+
+\item[nshare] This is an allocation which consists of one or more 
co-located {\em qshares}.  When 
+  exposed outside of RM this is usually thought of as a ``process''.  It 
means, literally,
+  ``n qshares''.  
+  
+Be careful, an {\em nshare} is NOT a process, it is an allocation that 
can be put to
+any use or to no use if desired.  The RM does not care what an {\em 
nshare} is 

svn commit: r1727979 [3/3] - in /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook: ./ images/ducc-internals/ part5/

[challngr]

Added: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-sm.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-sm.tex?rev=1727979=auto
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-sm.tex
 (added)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-sm.tex
 Mon Feb  1 17:36:08 2016
@@ -0,0 +1,578 @@
+% 
+% Licensed to the Apache Software Foundation (ASF) under one
+% or more contributor license agreements.  See the NOTICE file
+% distributed with this work for additional information
+% regarding copyright ownership.  The ASF licenses this file
+% to you under the Apache License, Version 2.0 (the
+% "License"); you may not use this file except in compliance
+% with the License.  You may obtain a copy of the License at
+% 
+%   http://www.apache.org/licenses/LICENSE-2.0
+% 
+% Unless required by applicable law or agreed to in writing,
+% software distributed under the License is distributed on an
+% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+% KIND, either express or implied.  See the License for the
+% specific language governing permissions and limitations
+% under the License.
+% 
+
+% \section{DUCC Service Manager}
+This section describes the architecture and internal structure of the
+DUCC Service Manager, referred to as the ``SM''.
+
+\section{Introduction}
+The SM function is to insure that any services needed by
+DUCC jobs are running and functional at the time they are needed by
+jobs.  Previous to the incarnation of the SM it was necessary for
+users to manually invoke the processes implementing their services.  If
+these processes were to crash, jobs dependent on them would stop until
+some human was able to restart the service.  If the operating system,
+or batch system supporting the jobs (DUCC, in our case) was to
+be restarted, users would again have to manually start the services.
+
+By ``registering'' a service with the SM, a user can trust DUCC to
+keep the service alive and functional across all manner of faults and
+system restarts.  As well, the SM has a mechanism for ``testing'' a
+service to determine if it is operational, and to inform the DUCC
+Web Server when it is not.  
+
+If a user submits a job that declares a dependency on a service, the SM
+is able to start the service as needed, and is able to stop the service
+when no longer needed, freeing resources.
+
+In essence, the SM can be thought of as a ``proxy user'' dedicated
+to insuring that services are always available when needed.
+
+\section{Architectural Overview}
+
+Figure ~\ref{fig:sm-structure} below shows the high-level object,
+threading, and process structure of SM and should be referenced
+while reading this document.
+
+The SM can be pictured as being composed of four major parts:
+\begin{enumerate}
+  \item Initialization and interaction with external components.
+External components include user requests and  other DUCC components 
such as
+the Orchestrator.
+  \item A ``Service Instance'' manager.  This part resolves
+dependencies on services, starts and stops service instances
+according to the needs of jobs and the policies declared in
+the service registries, and handles the service instance
+lifetimes.
+  \item A ``Service Health'' manager.  This part continually
+``tests'' services to determine whether they are 
+functional.  This is referred to as the ``pinger'' and the
+test is known as a ``ping''.
+  \item A ``CLI Handler'' which reacts to requests from users.
+\end{enumerate}
+
+\begin{figure}[H]
+  \centering
+  \includegraphics[width=5.5in]{images/ducc-internals/sm-structure.png}
+  \caption{Service Manager Structure}
+  \label{fig:sm-structure}
+\end{figure}
+
+   The terminology around Services can be confusing.  We review the ideas here.
+
+   There are three ``countable'' entities involved in services.  
+   \begin{description}
+ \item[Service Registration] When a service is ``registered'' the Service 
Manager assigns
+   a new, unique {\em Registration ID} to the registration.  This ID is 
associated with, and
+   remains with, the service throughout its lifetime and beyond when it is 
archived.
+
+ \item[Service Instance] When the Service Manager starts a service it 
issues a series of
+   ``submit'' orders to the Orchestrator, one for each {\em Service 
Instance}.  All these 
+   instances are associated with the {\em Service Registration}.  The 
orchestrator assigns
+   a unique ID to each service instance, which is also 

svn commit: r1727979 [1/3] - in /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook: ./ images/ducc-internals/ part5/

[challngr]

Author: challngr
Date: Mon Feb  1 17:36:08 2016
New Revision: 1727979

URL: http://svn.apache.org/viewvc?rev=1727979=rev
Log:
UIMA-4777 Internals documentation.

Added:

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/db-structure.png
   (with props)

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-1.png
   (with props)

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-2.png
   (with props)

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.png
   (with props)

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.vsd
   (with props)

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/sm-structure.png
   (with props)

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/internals-book.tex

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-database.tex

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-rm.tex

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-sm.tex

Added: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/db-structure.png
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/db-structure.png?rev=1727979=auto
==
Binary file - no diff available.

Propchange: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/db-structure.png
--
svn:mime-type = application/octet-stream

Added: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-1.png
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-1.png?rev=1727979=auto
==
Binary file - no diff available.

Propchange: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-1.png
--
svn:mime-type = application/octet-stream

Added: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-2.png
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-2.png?rev=1727979=auto
==
Binary file - no diff available.

Propchange: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-2.png
--
svn:mime-type = application/octet-stream

Added: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.png
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.png?rev=1727979=auto
==
Binary file - no diff available.

Propchange: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.png
--
svn:mime-type = application/octet-stream

Added: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.vsd
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.vsd?rev=1727979=auto
==
Binary file - no diff available.

Propchange: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.vsd
--
svn:mime-type = application/octet-stream

Added: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/sm-structure.png
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/sm-structure.png?rev=1727979=auto

svn commit: r1726063 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database: DbHandle.java DbManager.java HistoryManagerDb.java

[challngr]

Author: challngr
Date: Thu Jan 21 18:55:04 2016
New Revision: 1726063

URL: http://svn.apache.org/viewvc?rev=1726063=rev
Log:
UIMA-4577 Watch out for nulls in the OR history objects.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java?rev=1726063=1726062=1726063=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
 Thu Jan 21 18:55:04 2016
@@ -76,7 +76,15 @@ public class DbHandle
BoundStatement bound = boundStatement.bind(fields);
return execute(bound);
 } finally {
-   if ( manager.noisy ) logger.info(methodName, null, 
"Time to execute prepared statement:", ps.getQueryString(), 
System.currentTimeMillis() - now);
+   if ( manager.noisy ) {
+logger.info(methodName, null, "Time to execute prepared 
statement:", ps.getQueryString(), System.currentTimeMillis() - now);
+StringBuffer buf = new StringBuffer("Fields for statement: ");
+for ( Object o: fields ) {
+buf.append(o.toString());
+buf.append(" ");
+}
+logger.info(methodName, null, buf.toString());
+}
}
 }
 

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java?rev=1726063=1726062=1726063=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
 Thu Jan 21 18:55:04 2016
@@ -43,12 +43,12 @@ import com.datastax.driver.core.policies
  */
 public class DbManager
 {
-static final String URL_PROPERTY = "ducc.database.url";
+static final String URL_PROPERTY = "ducc.database.host";
 static final String NOISE_PROPERTY = "ducc.database.noisy";
 private static String db_id = null;
 private static String db_pw = null;
 
-boolean noisy = false;
+boolean noisy = true;
 String dburl;
 DuccLogger logger;
 

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java?rev=1726063=1726062=1726063=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
 Thu Jan 21 18:55:04 2016
@@ -300,7 +300,7 @@ public class HistoryManagerDb
 IDuccSchedulingInfo dsx = j.getSchedulingInfo();
 
 String user = dsi.getUser();
-String jclass = dsx.getSchedulingClass();
+String jclass = getString(dsx.getSchedulingClass());
 
 int memory = toInt(dsx.getMemorySizeRequested());
 long submission = dsi.getDateOfSubmissionMillis();
@@ -354,9 +354,9 @@ public class HistoryManagerDb
 
 long ducc_pid = idp.getDuccId().getFriendly();
 long pid = toInt(idp.getPID());
-String node = idp.getNodeIdentity().getName();
-String reason_agent = 
idp.getReasonForStoppingProcess(); // called "reason" in duccprocess but not in 
ws
-String reason_scheduler = 
idp.getProcessDeallocationType().toString(); // called 
"processDeallocationType" in duccprocess but not in ws
+String node = 
getString(idp.getNodeIdentity().getName());
+String reason_agent = 
getS

svn commit: r1725763 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler: Machine.java NodePool.java Scheduler.java Share.java

[challngr]

Author: challngr
Date: Wed Jan 20 17:42:22 2016
New Revision: 1725763

URL: http://svn.apache.org/viewvc?rev=1725763=rev
Log:
UIMA-4742 Clean-up and adjust rules for purging a node for vary-on and for node 
death, first delivery.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java?rev=1725763=1725762=1725763=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
 Wed Jan 20 17:42:22 2016
@@ -444,19 +444,19 @@ public class Machine
 return ret;
 }
 
-RmQueriedMachine queryOfflineMachine()  // UIMA-4234
-{
-RmQueriedMachine ret = queryMachine();
-ret.setOffline();
-return ret;
-}
-
-RmQueriedMachine queryUnresponsiveMachine()// UIMA-4234
-{
-RmQueriedMachine ret = queryMachine();
-ret.setUnresponsive();
-return ret;
-}
+//RmQueriedMachine queryOfflineMachine()  // UIMA-4234
+//{
+//RmQueriedMachine ret = queryMachine();
+//ret.setOffline();
+//return ret;
+//}
+//
+//RmQueriedMachine queryUnresponsiveMachine()// UIMA-4234
+//{
+//RmQueriedMachine ret = queryMachine();
+//ret.setUnresponsive();
+//return ret;
+//}
 
 /**
  * A machine's investment is the sum of it's share's investments.

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java?rev=1725763=1725762=1725763=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
 Wed Jan 20 17:42:22 2016
@@ -542,13 +542,6 @@ class NodePool
 Machine m = allMachines.get(n);
 
 if ( m == null ) {
-m = unresponsiveMachines.get(n);
-}
-if ( m == null ) {
-m = offlineMachines.get(n);
-}
-
-if ( m == null ) {
 for ( NodePool np : children.values() ) {
 m = np.getMachine(n);
 if ( m != null ) {
@@ -972,36 +965,36 @@ class NodePool
 return np;
 }
 
-private synchronized void incrementOnlineByOrder(int order)
-{
-if ( ! onlineMachinesByOrder.containsKey(order) ) {
-onlineMachinesByOrder.put(order, 1);
-} else {
-onlineMachinesByOrder.put(order, onlineMachinesByOrder.get(order) 
+ 1);
-}
-}
-
-private synchronized void decrementOnlineByOrder(int order)
-{
-onlineMachinesByOrder.put(order, onlineMachinesByOrder.get(order) - 1);
-}
-
-synchronized void getLocalOnlineByOrder(int[] ret) // for queries, 
just me
-{
-for ( int o: onlineMachinesByOrder.keySet() ) {
-ret[o] += onlineMachinesByOrder.get(o);
-}
-}
-
-synchronized void getOnlineByOrder(int[] ret) // for queries
-{
-for ( int o: onlineMachinesByOrder.keySet() ) {
-ret[o] += onlineMachinesByOrder.get(o);
-}
-for ( NodePool child : children.values() ) {
-child.getOnlineByOrder(ret);
-}
-}
+//private synchronized void incrementOnlineByOrder(int order)
+//{
+//if ( ! onlineMachinesByOrder.containsKey(order) ) {
+//onlineMachinesByOrder.put(order, 1);
+//} else {
+//onlineMachinesByOrder.put(order, 
onlineMachinesByOrder.get(order) + 1);
+//}
+//}
+
+//private synchronized void decrementOnlineByOrder(int order)
+//{
+//onlineMachinesByOrder.put(order, onlineMachinesByOrder.get(order) - 
1);
+//}
+
+//synchronized void getLocalOnlineByOrder(int[] ret) // for 
queries, just me
+//{
+//for ( int o: onlineMachinesByOrder.keySet() ) {
+//ret[o

svn commit: r1724848 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler: Machine.java NodepoolScheduler.java

[challngr]

Author: challngr
Date: Fri Jan 15 16:34:33 2016
New Revision: 1724848

URL: http://svn.apache.org/viewvc?rev=1724848=rev
Log:
UIMA-4577 Must update db when reassigning share under defrag.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java?rev=1724848=1724847=1724848=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
 Fri Jan 15 16:34:33 2016
@@ -335,6 +335,13 @@ public class Machine
 this.virtual_share_order = share_order - blacklisted_shares;
 }
 
+public void reassignShare(Share s, IRmJob newjob)
+{
+removeShare(s);
+s.reassignJob(newjob);
+assignShare(s);
+}
+
 public void assignShare(Share s)
 {
String methodName = "assignShare";
@@ -345,9 +352,9 @@ public class Machine
 // Not transactional.  If this turns into a problem we'll have to 
find a way
persistence.setNodeProperties(id, RmNodes.Assignments, 
activeShares.size(), RmNodes.NPAssignments, countNpShares(), 
RmNodes.SharesLeft, shares_left);
persistence.addAssignment(id, s.getJob().getId(), s, 
getQuantum(), s.getJob().getShortType()); // update jobs on machine and 
specific shares
-logger.info(methodName, null, "Time to assign share in db", 
System.currentTimeMillis() - now);
+logger.info(methodName, s.getJob().getId(), "Time to assign 
share", s.getId(), "in db", System.currentTimeMillis() - now);
} catch (Exception e) {
-logger.warn(methodName, null, "Cannot save state; shares_left", 
shares_left, e);
+logger.warn(methodName, s.getJob().getId(), "Cannot save state for 
share", s.getId(), "shares_left", shares_left, e);
}
 
 }
@@ -364,9 +371,9 @@ public class Machine
 // Not transactional.  If this turns into a problem we'll have to 
find a way
persistence.setNodeProperties(id, RmNodes.Assignments,  
activeShares.size(), RmNodes.NPAssignments, countNpShares(),  
RmNodes.SharesLeft, shares_left);
persistence.removeAssignment(id, s.getJob().getId(), 
s);  // update jobs on machine and specific shares
-logger.info(methodName, null, "Time to remove share in db", 
System.currentTimeMillis() - now);
+logger.info(methodName, s.getJob().getId(), "Time to remove 
share", s.getId(), "in db", System.currentTimeMillis() - now);
} catch (Exception e) {
-logger.warn(methodName, null, "Cannot save state; shares_left", 
shares_left);
+logger.warn(methodName, s.getJob().getId(), "Cannot save state for 
share", s.getId(), "shares_left", shares_left);
}
 }
 

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java?rev=1724848=1724847=1724848=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java
 Fri Jan 15 16:34:33 2016
@@ -1872,7 +1872,8 @@ public class NodepoolScheduler
 if ( s.isPending() ) {
 if (s.getShareOrder() == nj.getShareOrder() ) {
// same size, reassign it directly
 logger.debug(methodName, nj.getId(), "Reassign expanded 
share", s.toString(), "from", rich_j.getId());
-s.reassignJob(nj);
+Machine m = s.getMachine();
+m.reassignShare(s, nj);
 rich_j.cancelPending(s);
 nj.assignShare(s);
 return false;

svn commit: r1724512 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4: admin/admin-commands.tex admin/ducc-database.tex admin/ducc-properties.tex ducc-aguide.tex insta

[challngr]

Author: challngr
Date: Wed Jan 13 21:09:43 2016
New Revision: 1724512

URL: http://svn.apache.org/viewvc?rev=1724512=rev
Log:
UIMA-4745 Database updates to duccbookk.

Added:

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-database.tex
Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/ducc-aguide.tex

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex?rev=1724512=1724511=1724512=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
 Wed Jan 13 21:09:43 2016
@@ -35,6 +35,8 @@
 The command \ducchome/admin/start\_ducc is used to start DUCC processes. 
If run with no parameters
 it takes the following actions:
 \begin{itemize}
+  \item Starts the ActiveMQ server.
+  \item Starts the database.
   \item Starts the management processes Resource Manager, Orchestrator, 
Process Manager,  
   Services Manager, and Web Server on the local node (where start\_ducc is 
executed.   
   \item Starts an agent process on every node named in the default node 
list. 
@@ -76,6 +78,8 @@ start\_ducc -c sm -c pm -c rm -c or@bj22
 \item[sm]The Service Manager
 \item[ws]The Web Server
 \item[agent]Node Agents
+\item[broker] ActiveMQ broker
+\item[db] Database
   \end{description}
 
   \item[--nothreading] If specified, the command does not run in 
multi-threaded mode
@@ -138,9 +142,16 @@ start\_ducc -c sm -c pm -c rm -c or@bj22
 \label{subsec:admin.stop-ducc}
 
 \subsubsection{{\em Description:}}
-Stop\_ducc is used to stop DUCC processes. If run with no parameters it 
takes the following 
-actions:
-\todo Garbled by maven or docbook, update this
+Stop\_ducc is used to stop DUCC processes. At least one parameter is 
required.
+When {\em -a} is specified, the following actions are taken:
+\begin{itemize}
+   \item Uses the ActiveMQ broker to broadcast a shutdown request to all
+DUCC compoments, other than the ActiveMQ broker itself, and the 
database.
+  \item Waits a bit, for all daemons to stop.
+  \item Stops the database.
+  \item Stops the ActiveMQ broker.
+\end{itemize}
+
 
 \subsubsection{\em Usage:}
 
@@ -202,10 +213,19 @@ start_ducc -c rm
   \item[pm] The Process Manager. 
   \item[sm] The Service Manager. 
   \item[ws] The Web Server. 
+  \item[db] The database.
   \item[broker] The ActiveMQ broker (only if the broker is 
auto-managed).
   \item[agent\@node] Node Agent on the specified node.
   \end{description}
 
+  \item[-w, --wait {[time in seconds]}] If given, this signals the 
time to wait
+after broadcasting the shutdown signal, and before stopping the 
ActiveMQ broker itself.
+If not specified, the default is 60 seconds.  
+
+NOTE: In production systems, it is generally wise to use the 
default of 60 seconds.  For
+test systems a shorter wait speeds cycle time.  Be sure to use 
{\em check\_ducc -k} after
+{\em stop\_ducc} if you change the wait time to insure all 
processes are actually stopped.
+
   \item[--nothreading] If specified, the command does not run in 
multi-threaded mode
 even if it is supported on the local platform.
   
@@ -595,3 +615,90 @@ Nodepool power
 
 \paragraph{Notes:}
 None.
+
+\subsection{db\_create}
+\label{subsec:cli.db.create}
+
+\paragraph{Description:}
+This command is used to initialize the database.  Normally the 
database is initialized
+during {\em ducc\_post\_install} but if this is an existing DUCC 
installation that is 
+being migrated from a version that does not use the database, it will 
be necessary to
+initialize the database with this command.
+
+This command performs the following actions:
+\begin{enumerate}
+  \item Starts the database.
+  \item Disables the default database superuser.
+  \item Installs a database superuser as ``ducc'' and sets

svn commit: r1723398 - in /uima/sandbox/uima-ducc/trunk: src/main/admin/ uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orche

[challngr]

Author: challngr
Date: Wed Jan  6 18:41:57 2016
New Revision: 1723398

URL: http://svn.apache.org/viewvc?rev=1723398=rev
Log:
UIMA-4577 Add tables for job process etc details.

Added:
uima/sandbox/uima-ducc/trunk/src/main/admin/q_processes   (with props)
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java

uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java

uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccSchedulingInfo.java

uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py?rev=1723398=1723397=1723398=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py Wed Jan  6 18:41:57 
2016
@@ -11,6 +11,83 @@ def execute(CMD):
 print CMD
 return os.system(CMD)
 
+# 

+# these next methods are used to parse a table returned from cqlsh into a
+#   - header
+#   - dictionary of values for each row
+
+# parse the header into a list of names
+def parse_header(header):
+ret = []
+parts = header.split('|')
+for p in parts:
+ret.append(p.strip())
+return ret
+
+# parse a single line into a dictionary with key from the header and value 
from the line
+def parse_line(header, line):
+parts = line.split('|')
+ret = {}
+for k, v in zip(header, parts):
+ret[k] = v.strip()
+return ret
+
+# parse a set of lines returned from cqlsh into a header and a list of 
dictionaries, one per line
+# header_id is a sting we use to positively identify a header line
+def parse(lines, header_id):
+ret = []
+header = []
+for l in lines:
+l = l.strip()
+# print '[]', l
+if ( l == '' ):
+continue
+if ( '---' in l ):
+continue;
+if ( 'rows)' in l ):
+continue;
+if ( header_id in l ):
+header = parse_header(l)
+continue
+
+ret.append(parse_line(header, l))
+
+return header, ret
+
+# given a header and a collection of lines parsed by the utilities above, 
print a 
+# mostly un-ugly listing of the table retults
+def format(header, lines):
+
+# calculate max column widths
+hlens = {}
+for k in header:
+hlens[k] = len(k)
+for line in lines:
+if ( not hlens.has_key(k) ):
+hlens[k] = len(line[k])
+else:
+hlens[k] = max(len(line[k]), hlens[k])
+
+# create a format string from the widths 
+fmt = ''
+for k in header:
+fmt = fmt + ' %' + str(hlens[k]) + 's' 
+
+# first the header
+print fmt % tuple(header)
+
+# now the rows
+for line in lines:
+l = []
+for k in header:
+l.append(line[k])
+print fmt % tuple(l)
+return
+
+
+# end of row parsing utilities
+# 

+
 def stop_database(pidfile):
 print "Stopping the dtabase."
 

Added: uima/sandbox/uima-ducc/trunk/src/main/admin/q_processes
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/q_processes?rev=1723398=auto
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/q_processes (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/q_processes Wed Jan  6 18:41:57 
2016
@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+# ---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, V

svn commit: r1721195 - /uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java

[challngr]

Author: challngr
Date: Mon Dec 21 15:34:18 2015
New Revision: 1721195

URL: http://svn.apache.org/viewvc?rev=1721195=rev
Log:
UIMA-4712 - set service id for horizontal AP/Service allocation

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java?rev=1721195=1721194=1721195=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
 Mon Dec 21 15:34:18 2015
@@ -108,6 +108,17 @@ public class JobManagerConverter
 }
 }
   
+// UIMA-4712
+long toLong(String s, long deflt)
+{
+try {
+long val = Long.parseLong(s);
+return ( val == 0L ) ? deflt : val;
+} catch ( Throwable t ) {
+return deflt;
+}
+}
+  
 void refuse(IRmJob j, String reason)
 {
 j.refuse(reason);
@@ -467,13 +478,6 @@ public class JobManagerConverter
  * Convert a JobManager Job into a ResourceManager RmJob.  We assume this 
job is NOT in
  * our lists.
  *
- * NOTE IMPORTANT NOTE
- *
- *Until Lou's job contains all required scheduling fields I do a 
conversion that enhances
- *what I receive with defaults so the scheduler can actually schedule 
the job.
- *
- * NOTE IMPORTANT NOTE
- *
  * @param job
  */
 boolean jobArrives(IDuccWork job)
@@ -514,6 +518,7 @@ public class JobManagerConverter
 String user_name  = sti.getUser().trim();
 j.setUserName(user_name);
 j.setJobName(name);
+j.setServiceId(toLong(job.getServiceId(), 0L)); // UIMA-4712 only 
non-zero on actual service instances 
 
 int threads   = toInt(si.getThreadsPerProcess(), 
scheduler.getDefaultNThreads());
 int user_priority = toInt(si.getSchedulingPriority(), 100);

svn commit: r1719426 - /uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy

[challngr]

Author: challngr
Date: Fri Dec 11 14:18:06 2015
New Revision: 1719426

URL: http://svn.apache.org/viewvc?rev=1719426=rev
Log:
UIMA-4577 Redo rm_qoccupancy using cqlsh because it's so much faster than 
  spawning java!

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy?rev=1719426=1719425=1719426=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy Fri Dec 11 
14:18:06 2015
@@ -37,12 +37,12 @@ class DuccRmQOccupancy(DuccUtil):
 status = 'up'
 else:
 status = 'down'
-print "%20s %11s %6s %6s %15s %10s %3s(Q) %6s %6s %8s %7s %10s  
%8s" %  (n['name'], n['blacklisted'], n['online'], status, n['nodepool'], 
+print "%20s %11s %6s %6s %15s %10s %3s(Q) %6s %6s %8s %7s %10s  
%-8s" %  (n['name'], n['blacklisted'], n['online'], status, n['nodepool'], 
  
n['memory'], n['share_order'], n['shares_left'], n['assignments'], 
  
n['np_assignments'], n['quantum'], n['reservable'], n['classes'])
 if ( shares.has_key(n['name']) ):
 for s in shares[n['name']]:
-fmt = '%19s ' + s['jobtype'] +'[%8d] S[%8d] O[%d] II[%8d] 
IR[%8d] E[%5s] P[%5s] F[%5s] S[%10s]'
+fmt = '%19s ' + s['jobtype'] +'[%8s] S[%8s] O[%s] II[%8s] 
IR[%8s] E[%5s] P[%5s] F[%5s] S[%10s]'
 state = s['state']
 if ( state == 'null' ):
 state = "Assigned"
@@ -50,13 +50,66 @@ class DuccRmQOccupancy(DuccUtil):
 
 print ''
 
-
-# Given DUCC_HOME, a directory, and part of the name of a jar, find the 
actual name of the jar which will
-# likeley be versioned
-def resolve_jar(self, DH, dirname, basename):
-partial = DH + '/' + dirname + '/'+ basename + '*'
-ret = glob.glob(partial)
-return ret[0]
+
+def parse_header(self, header):
+ret = []
+parts = header.split('|')
+for p in parts:
+ret.append(p.strip())
+return ret
+
+def parse_node(self, header, line):
+parts = line.split('|')
+ret = {}
+for k, v in zip(header, parts):
+ret[k] = v.strip()
+return ret
+
+def parse_share(self, header, line):
+parts = line.split('|')
+ret = {}
+for k, v in zip(header, parts):
+ret[k] = v.strip()
+return ret
+
+
+def rmnodes(self, lines):
+nodes = []
+shares = {}
+header = []
+for l in lines:
+l = l.strip()
+# print '[]', l
+if ( l == '' ):
+continue
+if ( '---' in l ):
+continue;
+if ( 'rows)' in l ):
+continue;
+if ( 'assignments' in l ):
+doing_nodes = True
+doing_shares = False
+header = self.parse_header(l)
+continue
+if ( 'investment' in l ):
+doing_nodes = False
+doing_shares = True
+header = self.parse_header(l)
+continue
+if ( doing_nodes ):
+nodes.append(self.parse_node(header, l))
+continue
+if ( doing_shares ):
+s = self.parse_share(header, l)
+k = s['node']
+if ( shares.has_key(k) ):
+share_list = shares[k]
+else:
+share_list = []
+shares[k] = share_list
+share_list.append(s)
+continue
+return nodes, shares
 
 def main(self, argv):
 
@@ -65,53 +118,20 @@ class DuccRmQOccupancy(DuccUtil):
 sys.exit(1);
 
 DH = self.DUCC_HOME
-CP = [self.resolve_jar(DH, '/lib/uima-ducc', 'uima-ducc-database'),
-  self.resolve_jar(DH, '/lib/uima-ducc', 'uima-ducc-common'),
-  DH + '/lib/cassandra/*',
-  DH + '/lib/apache-log4j/*',
-  DH + '/lib/guava/*',
-  self.resolve_jar(DH, '/cassandra-server/lib', 'slf4j-api'),
-  self.resolve_jar(DH, '/apache-uima/apache-activemq/lib', 
'slf4j-log4j12'),
- ]
-os.environ['CLASSPATH'] = ':'.join(CP)
-
-DUCC_JVM_OPTS = ''
-DUCC_JVM_OPTS = DUCC_JVM_OPTS + ' -DDUCC_HOME=' + self.DUCC_HOME
-DUCC_JVM_OPTS = DUCC_JVM_OPTS + ' 
-Dducc.rm.persistence.impl=org.apache.uima.du

svn commit: r1719195 - in /uima/sandbox/uima-ducc/trunk: src/main/admin/ uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/ uima-ducc-database/src/main/java/org/apache/uima/duc

[challngr]

Author: challngr
Date: Thu Dec 10 20:13:26 2015
New Revision: 1719195

URL: http://svn.apache.org/viewvc?rev=1719195=rev
Log:
UIMA-4577 Tweaks so database classes can be used in CLI utilities.  Implement 
rm_qoccupancy using
  the DB as proof of concept.

Added:

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmQLoad.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmShareState.java
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy?rev=1719195=1719194=1719195=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy Thu Dec 10 
20:13:26 2015
@@ -21,59 +21,95 @@
 
 import os
 import sys
+import glob
 import subprocess
 
 from ducc_util  import DuccUtil
 
 class DuccRmQOccupancy(DuccUtil):
 
-def format_shares(self, sh):
-for s in sh:
-if ( s['blacklisted'] ):
-pass
-else:
-print '%19s J[%8d] S[%8d] O[%d] II[%8d] IR[%8d] E[%5s] P[%5s] 
F[%5s] I[%5s]' %  ('', s['jobid'], s['shareid'], s['order'], 
s['investment-init'], s['investment-run'], 
- s['evicted'], s['purged'], s['fixed'], s['initialized'])
-
-
-def format_machines(self, lines):
-
-print("%20s %11s %6s %6s %15s %10s %5s %6s" %  ("Node", "Blacklisted", 
"Online", "Status", "Nodepool", "Memory", "Order", "Free"))
-
-for m in lines:
-print "%20s %11s %6s %6s %15s %10s %5s %6s" %  (m['name'], 
m['blacklisted'], m['online'], m['status'], m['nodepool'], m['memory'], 
m['order'], m['shares-free'])
-if ( len(m['shares']) != 0 ) :
-self.format_shares(m['shares'])
-print ''
 
+def format(self, nodes, shares):
+print("%20s %11s %6s %6s %15s %10s %6s %6s %6s %8s %7s %10s %8s" %  
("Node", "Blacklisted", "Online", "Status", "Nodepool", "Memory", "Order", 
"Free", "In-Use", "Np-InUse", "Quantum", "Reservable", "Classes"))
+print("%20s %11s %6s %6s %15s %10s %6s %6s %6s %8s %7s %10s %8s" %  
("", "---", "--", "--", "", "--", "-", 
"", "--", "", "---", "--", "---"))
+for n in nodes:
+if (n['responsive']):
+status = 'up'
+else:
+status = 'down'
+print "%20s %11s %6s %6s %15s %10s %3s(Q) %6s %6s %8s %7s %10s  
%8s" %  (n['name'], n['blacklisted'], n['online'], status, n['nodepool'], 
+ 
n['memory'], n['share_order'], n['shares_left'], n['assignments'], 
+ 
n['np_assignments'], n['quantum'], n['reservable'], n['classes'])
+if ( shares.has_key(n['name']) ):
+for s in shares[n['name']]:
+fmt = '%19s ' + s['jobtype'] +'[%8d] S[%8d] O[%d] II[%8d] 
IR[%8d] E[%5s] P[%5s] F[%5s] S[%10s]'
+state = s['state']
+if ( state == 'null' ):
+state = "Assigned"
+print fmt %  ('', s['job_id'], s['ducc_dbid'], 
s['share_order'], s['init_time'], s['investment'], s['evicted'], s['purged'], 
s['fixed'], state)
 
-def format(self, lines):
-qoccupancy = eval(lines)
-self.format_machines(qoccupancy)
+print ''
 
 
+# Given DUCC_HOME, a directory, and part of the name of a jar, find th

svn commit: r1718459 - in /uima/sandbox/uima-ducc/trunk: src/main/admin/ src/main/resources/ uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ uima-ducc-database/src/main/java/org/apache/uima/ducc

[challngr]

Author: challngr
Date: Mon Dec  7 20:41:22 2015
New Revision: 1718459

URL: http://svn.apache.org/viewvc?rev=1718459=rev
Log:
UIMA-4577. Multiple minor tweaks. Fix RM vary/on-off with node death issue.  
Change ducc.properties
   for db from ducc.state.database.url to simple ducc.database.url.  
Fix automatic config
   of DB URL in cassandra.yaml.

Removed:

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ReadCkpt.java
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml
uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties

uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccMonitor.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbVerify.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1718459=1718458=1718459=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Mon Dec  7 
20:41:22 2015
@@ -205,7 +205,7 @@ class DuccUtil(DuccBase):
 if ( self.db_bypass == True ):
 return True
 
-dbnode = self.ducc_properties.get('ducc.state.database.url')
+dbnode = self.ducc_properties.get('ducc.database.host')
 if ( dbnode == None ):
 print 'No database location defined.'
 return False
@@ -235,7 +235,7 @@ class DuccUtil(DuccBase):
 return True
 
 print 'Starting database'
-dbnode = self.ducc_properties.get('ducc.state.database.url')
+dbnode = self.ducc_properties.get('ducc.database.host')
 dbu.update_cassandra_config(self.DUCC_HOME, dbnode)
 
 max_attempts = 5

Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml?rev=1718459=1718458=1718459=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml Mon Dec  7 
20:41:22 2015
@@ -274,7 +274,7 @@ seed_provider:
   parameters:
   # seeds is actually a comma-delimited list of addresses.
   # Ex: ",,"
-  - seeds: "bluej538"
+  - seeds: "DUCCHOST"
 
 # For workloads with more data than can fit in memory, Cassandra's
 # bottleneck will be reads that need to fetch data from
@@ -387,7 +387,7 @@ ssl_storage_port: 7001
 # you can specify which should be chosen using listen_interface_prefer_ipv6. 
If false the first ipv4
 # address will be used. If true the first ipv6 address will be used. Defaults 
to false preferring
 # ipv4. If there is only one address it will be selected regardless of 
ipv4/ipv6.
-listen_address: bluej538
+listen_address: DUCCHOST
 # listen_interface: eth0
 # listen_interface_prefer_ipv6: false
 
@@ -445,7 +445,7 @@ start_rpc: true
 # you can specify which should be chosen using rpc_interface_prefer_ipv6. If 
false the first ipv4
 # address will be used. If true the first ipv6 address will be used. Defaults 
to false preferring
 # ipv4. If there is only one address it will be selected regardless of 
ipv4/ipv6.
-rpc_address: bluej538
+rpc_address: DUCCHOST
 # rpc_interface: eth1
 # rpc_interface_prefer_ipv6: false
 

Modified: 
uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties?re

svn commit: r1718491 - in /uima/sandbox/uima-ducc/trunk: uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/ uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ uima

[challngr]

Author: challngr
Date: Mon Dec  7 23:08:53 2015
New Revision: 1718491

URL: http://svn.apache.org/viewvc?rev=1718491=rev
Log:
UIMA-4577. Add np_assignments (non-preemptable assignments) to rmnodes table.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java?rev=1718491=1718490=1718491=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
 Mon Dec  7 23:08:53 2015
@@ -266,6 +266,10 @@ public interface IRmPersistence
 public String pname() { return "assignments"; }
 public Type type()  { return Type.Integer; }
 },
+NPAssignments {
+public String pname() { return "np_assignments"; }
+public Type type()  { return Type.Integer; }
+},
 ;
 public boolean isPrimaryKey() { return false; }
 public boolean isPrivate(){ return false; }

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java?rev=1718491=1718490=1718491=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java
 Mon Dec  7 23:08:53 2015
@@ -65,7 +65,7 @@ public class RmNodeState
 System.out.println("Usage: RmNodeState ");
 System.exit(1);
 }
-System.setProperty("ducc.state.database.url", args[0]);
+System.setProperty(DbManager.URL_PROPERTY, args[0]);
 
 RmNodeState rns = new RmNodeState(args[0]);
 try {

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java?rev=1718491=1718490=1718491=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
 Mon Dec  7 23:08:53 2015
@@ -243,6 +243,17 @@ public class Machine
 return answer;
 }
 
+public int countNpShares()
+{
+int ret = 0;
+for ( Share s : activeShares.values() ) {
+if ( s.isFixed() ) {
+ret += s.getShareOrder();
+}
+}
+return ret;
+}
+
 public int countProcesses()
 {
 return activeShares.size();
@@ -319,7 +330,7 @@ public class Machine
 shares_left -= s.getShareOrder();
 try {
 // Not transactional.  If this turns into a problem we'll have to 
find a way
-   persistence.setNodeProperties(id, RmNodes.Assignments, 
activeShares.size(), RmNodes.SharesLeft, shares_left);
+   persistence.setNodeProperties(id, RmNodes.Assignments, 
activeShares.size(), RmNodes.NPAssignments, countNpShares(), 
RmNodes.SharesLeft, shares_left);
persistence.addAssignment(id, s.getJob().getId(), s, 
getQuantum(), s.getJob().getShortType()); // update jobs on machine and 
specific shares
 logger.info(methodName, null, "Time to assign share in db", 
System.currentTimeMillis() - now);
} catch (Exception e) {
@@ -338,7 +349,7 @@ public class Machine
 shares_left += s.getShareOrder();
 try {
 // Not transactional.  If this turns into a problem we'll have to 
find a way
-   persistence.setNodeProperties(id, RmNodes.Assignments, 
ac

svn commit: r1717665 - /uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader

[challngr]

Author: challngr
Date: Wed Dec  2 18:34:43 2015
New Revision: 1717665

URL: http://svn.apache.org/viewvc?rev=1717665=rev
Log:
UIMA-4577 db_loader starts and stops the db for you to do the load

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader?rev=1717665=1717664=1717665=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader Wed Dec  2 18:34:43 
2015
@@ -47,10 +47,6 @@ class DbLoader(DuccUtil):
 print 'db_loader -i in [--no-archive]'
 print ''
 print 'Where:'
-print '-d'
-print ' Specifies the database must be dropped and 
reinitialized if it already exists.'
-print ' If not specified, the correct schema must already 
exist.  Object already in'
-print ' the database are not overwritten.'
 print '-i in'
 print ' Specifies the DUCC_HOME of the DUCC runtime to be 
moved into the database.'
 print ''
@@ -68,7 +64,6 @@ class DbLoader(DuccUtil):
 
 def main(self, argv):
 
-drop= False
 in_home = None
 out_url = None
 archive = True
@@ -80,8 +75,6 @@ class DbLoader(DuccUtil):
 for ( o, a ) in opts:
 if o in ('-i'):
 in_home = a
-elif o in ('-d'):
-drop = True
 elif o in ('--no-archive'):
 archive = False
 else:
@@ -106,9 +99,6 @@ class DbLoader(DuccUtil):
 if ( not archive ):
 DUCC_JVM_OPTS = DUCC_JVM_OPTS + ' -DDONT_ARCHIVE'
 
-if ( drop ):
-DUCC_JVM_OPTS = DUCC_JVM_OPTS + ' -DDROP_DATABASE'
-
 CMD = [self.java(), DUCC_JVM_OPTS, 
'org.apache.uima.ducc.database.DbLoader', in_home, out_url]
 CMD = ' '.join(CMD)
 

svn commit: r1717663 - /uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader

[challngr]

Author: challngr
Date: Wed Dec  2 18:25:53 2015
New Revision: 1717663

URL: http://svn.apache.org/viewvc?rev=1717663=rev
Log:
UIMA-4577 db_loader starts and stops the db for you to do the load

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader?rev=1717663=1717662=1717663=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader Wed Dec  2 18:25:53 
2015
@@ -35,7 +35,7 @@ from ducc_util import DuccUtil
 class DbLoader(DuccUtil):
 
 def __init__(self):
-DuccUtil.__init__(self)
+DuccUtil.__init__(self, True)
 
 def usage(self, *args):
 if ( args[0] != None ):
@@ -44,7 +44,7 @@ class DbLoader(DuccUtil):
 print ''
 print 'Usage:'
 print ''
-print 'db_loader -i in -o out [--no-archive]'
+print 'db_loader -i in [--no-archive]'
 print ''
 print 'Where:'
 print '-d'
@@ -53,8 +53,7 @@ class DbLoader(DuccUtil):
 print ' the database are not overwritten.'
 print '-i in'
 print ' Specifies the DUCC_HOME of the DUCC runtime to be 
moved into the database.'
-print '-o dburl'
-print ' Specifies the database URL to contact to store the 
items.'
+print ''
 print '--no-archive'
 print ' Suppresses archival of the input files.'
 print ''
@@ -83,8 +82,6 @@ class DbLoader(DuccUtil):
 in_home = a
 elif o in ('-d'):
 drop = True
-elif o in ('-o'):
-out_url = a
 elif o in ('--no-archive'):
 archive = False
 else:
@@ -93,9 +90,12 @@ class DbLoader(DuccUtil):
 
 if ( in_home == None ):
 self.usage("Missing input DUCC_HOME")
+
+out_url = self.ducc_properties.get('ducc.database.host')
 if ( out_url == None ):
-self.usage("Missing database URL");
+self.usage("Cannot find 'ducc.database.host' in your properties 
file.");
 
+self.db_start()
 
 if ( not self.db_alive(3) ):
 print "Database is not running or cannot be contacted."
@@ -116,6 +116,9 @@ class DbLoader(DuccUtil):
 print 'CLASSPATH', os.environ['CLASSPATH']
 print 'Executing', CMD
 os.system(CMD)
+
+
+self.db_stop()
 
 if __name__ == "__main__":
 console = DbLoader()

svn commit: r1717540 - in /uima/sandbox/uima-ducc/trunk: src/main/resources/ uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ uima-ducc-transport/src/main/java/org/apache/uima/ducc/tran

[challngr]

Author: challngr
Date: Tue Dec  1 21:44:49 2015
New Revision: 1717540

URL: http://svn.apache.org/viewvc?rev=1717540=rev
Log:
UIMA-4577 Retry DbCreate with supplied id/pw if default fail, to allow rerun of 
DbCreate.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java

Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml?rev=1717540=1717539=1717540=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml Tue Dec  1 
21:44:49 2015
@@ -75,7 +75,7 @@ authenticator: PasswordAuthenticator
 # - AllowAllAuthorizer allows any action to any user - set it to disable 
authorization.
 # - CassandraAuthorizer stores permissions in system_auth.permissions table. 
Please
 #   increase system_auth keyspace replication factor if you use this 
authorizer.
-authorizer: AllowAllAuthorizer
+authorizer: CassandraAuthorizer
 
 # Validity period for permissions cache (fetching permissions can be an
 # expensive operation depending on the authorizer, CassandraAuthorizer is

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java?rev=1717540=1717539=1717540=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
 Tue Dec  1 21:44:49 2015
@@ -39,12 +39,13 @@ public class DbCreate
 static final String DUCC_KEYSPACE = "ducc";
 static final String PASSWORD_KEY  = "db_password";
 static final String PASSWORD_FILE = "ducc.private.properties";
-static final int RETRY = 10;
+int RETRY = 10;
 
 DuccLogger logger = null;
 String dburl;
 String adminid = null;
 String adminpw = null;
+boolean useNewPw = false;
 
 private Cluster cluster;
 private Session session = null;
@@ -84,14 +85,16 @@ public class DbCreate
 
 // If we're here, we must first of all get rid of the cassandra su and 
set up our own
 
-AuthProvider auth = new PlainTextAuthProvider("cassandra", 
"cassandra");
+
 for ( int i = 0; i < RETRY; i++ ) {
 try {
+// First time, we nuke the default id / pw and install our own.
+AuthProvider auth = new PlainTextAuthProvider("cassandra", 
"cassandra");
 cluster = Cluster.builder()
 .withAuthProvider(auth)
 .addContactPoint(dburl)
 .build();
-
+
 session = cluster.connect();
 session.execute("CREATE USER IF NOT EXISTS " + adminid + " 
with password '" + adminpw + "' SUPERUSER");
 cluster.close();
@@ -108,14 +111,28 @@ public class DbCreate
 session.execute("ALTER USER cassandra  with password '" + 
uglypw + "' NOSUPERUSER");
 doLog(methodName, "Changed default super user's password and 
revoked its superuser authority.");
 doLog(methodName, "From this point, this DB can only be 
accessed in super user mode by user 'ducc'");
-
 break;
 } catch ( NoHostAvailableException e ) {
 doLog("Waiting for database to boot ...");
 session = null;
 cluster = null;
 } catch ( AuthenticationException e ) {
-doLog("Waiting for default authentication ...");
+// The default userid/pw failed, so we try again with the 
user-supplied one
+RETRY += i; // we'll extend the retry for a bit in 
case db took a while to start
+doLog(me

svn commit: r1717503 - in /uima/sandbox/uima-ducc/trunk: src/main/resources/ uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/ uima-ducc-database/src/main/java/org/apache/uima

[challngr]

Author: challngr
Date: Tue Dec  1 19:21:52 2015
New Revision: 1717503

URL: http://svn.apache.org/viewvc?rev=1717503=rev
Log:
UIMA-4577 Add RmLoad table and support.

Added:

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IDbJob.java
Modified:
uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IRmJob.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ISchedulerMain.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java

Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml?rev=1717503=1717502=1717503=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml Tue Dec  1 
19:21:52 2015
@@ -108,12 +108,12 @@ partitioner: org.apache.cassandra.dht.Mu
 # the configured compaction strategy.
 # If not set, the default directory is $CASSANDRA_HOME/data/data.
 data_file_directories:
-- DUCC/data
+- ../state/database/data
 
 # commit log.  when running on magnetic HDD, this should be a
 # separate spindle than the data directories.
 # If not set, the default directory is $CASSANDRA_HOME/data/commitlog.
-commitlog_directory: DUCC/commitlog
+commitlog_directory: ../state/database/commitlog
 
 # policy for data disk failures:
 # die: shut down gossip and client transports and kill the JVM for any fs 
errors or
@@ -228,7 +228,7 @@ counter_cache_save_period: 7200
 
 # saved caches
 # If not set, the default directory is $CASSANDRA_HOME/data/saved_caches.
-saved_caches_directory: DUCC/saved_caches
+saved_caches_directory: ../state/database/saved_caches
 
 # commitlog_sync may be either "periodic" or "batch." 
 # 
@@ -274,7 +274,7 @@ seed_provider:
   parameters:
   # seeds is actually a comma-delimited list of addresses.
   # Ex: ",,"
-  - seeds: "DUCC_HEAD"
+  - seeds: "bluej538"
 
 # For workloads with more data than can fit in memory, Cassandra's
 # bottleneck will be reads that need to fetch data from
@@ -387,7 +387,7 @@ ssl_storage_port: 7001
 # you can specify which should be chosen using listen_interface_prefer_ipv6. 
If false the first ipv4
 # address will be used. If true the first ipv6 address will be used. Defaults 
to false preferring
 # ipv4. If there is only one address it will be selected regardless of 
ipv4/ipv6.
-listen_address: DUCC_HEAD
+listen_address: bluej538
 # listen_interface: eth0
 # listen_interface_prefer_ipv6: false
 
@@ -445,7 +445,7 @@ start_rpc: true
 # you can specify which should be chosen using rpc_interface_prefer_ipv6. If 
false the first ipv4
 # address will be used. If true the first ipv6 address will be used. Defaults 
to false preferring
 # ipv4. If there is only one address it will be selected regardless of 
ipv4/ipv6.
-rpc_address: DUCC_HEAD
+rpc_address: bluej538
 # rpc_interface: eth1
 # rpc_interface_prefer_ipv6: false
 

Added: 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IDbJob.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IDbJob.java?rev=1717503=auto
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IDbJob.java
 (added)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IDbJob.java
 Tue Dec  1 19:21:52 2015
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * dis

svn commit: r1715427 - in /uima/sandbox/uima-ducc/trunk: uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/ uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ uima

[challngr]

Author: challngr
Date: Fri Nov 20 21:48:11 2015
New Revision: 1715427

URL: http://svn.apache.org/viewvc?rev=1715427=rev
Log:
UIMA-4577 Support rmshares table.

Added:

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IDbShare.java
Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ISchedulerMain.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java

Added: 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IDbShare.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IDbShare.java?rev=1715427=auto
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IDbShare.java
 (added)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IDbShare.java
 Fri Nov 20 21:48:11 2015
@@ -0,0 +1,45 @@
+package org.apache.uima.ducc.common.persistence.rm;
+
+import org.apache.uima.ducc.common.Node;
+import org.apache.uima.ducc.common.NodeIdentity;
+import org.apache.uima.ducc.common.utils.id.DuccId;
+
+public interface IDbShare {
+
+
+   public abstract int getNodepoolDepth();
+
+   public abstract String getNodepoolId();
+
+   public abstract DuccId getId();
+
+   // UIMA-4142
+   public abstract boolean isBlacklisted();
+
+   // UIMA-4142
+   public abstract DuccId getBlJobId();
+
+   public abstract NodeIdentity getNodeIdentity();
+
+   public abstract Node getNode();
+
+   /**
+* The order of the share itself.
+*/
+   public abstract int getShareOrder();
+
+   /**
+* Returns only initialization time.  Eventually getInvestment() may 
take other things into
+* consideration so we separate these two (even though currently they 
do the same thing.)
+*/
+   public abstract long getInitializationTime();
+
+   public abstract void setInitializationTime(long millis);
+
+   public abstract void setFixed();
+
+   public abstract boolean isPurged();
+public abstract boolean isEvicted();
+public abstract boolean isFixed();
+
+}

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java?rev=1715427=1715426=1715427=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
 Fri Nov 20 21:48:11 2015
@@ -91,8 +91,9 @@ public interface IRmPersistence
  * @param id The node name
  * @param jobid The duccid of the job owning the new shoare
  * @param shareid The DuccId of the new share.
+ * @param quantum The scheduling quantum in GB used for this share.
  */
-public void addAssignment(String id, DuccId jobid, DuccId shareid);
+public void addAssignment(String id, DuccId jobid, IDbShare share, int 
quantum, String jobtype) throws Exception;
 
 /**
  * Remove a share from the machine.
@@ -100,7 +101,54 @@ public interface IRmPersistence
  * @param jobid The duccid of the job owning the new shoare
  * @param shareid The DuccId of the new share

svn commit: r1714825 - /uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java

[challngr]

Author: challngr
Date: Tue Nov 17 16:16:31 2015
New Revision: 1714825

URL: http://svn.apache.org/viewvc?rev=1714825=rev
Log:
UIMA-4577 Work around "A Service is a Job is an AP" thing to correctly 
categorize work.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java?rev=1714825=1714824=1714825=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
 Tue Nov 17 16:16:31 2015
@@ -170,16 +170,32 @@ public class HistoryManagerDb
String methodName = "saveWork";
 Long nowP =  System.currentTimeMillis();
 String type = null;
-if ( w instanceof IDuccWorkJob ) {
-type = "job";
-} else if ( w instanceof IDuccWorkReservation ) {
-type = "reservation";
-} else if ( w instanceof IDuccWorkService ) {
-type = "service";
-} else {
-   throw new IllegalArgumentException("Improper object passed to 
saveWork");
-}
 
+switch ( w.getDuccType() ) {
+case Job:
+type = "job";
+break;
+case Service:
+case Pop:
+switch ( ((IDuccWorkService)w).getServiceDeploymentType() ) 
+{
+case uima:
+case custom:
+type = "service";
+break;
+case other:
+type = "AP";
+break;
+}
+break;
+case Reservation:
+type = "reservation";
+break;
+default:
+// illegal - internal error if this happens
+logger.error(methodName, w.getDuccId(), "Unknown job type", 
w.getDuccType(), "Cannot save to database.");
+return;
+}
 logger.info(methodName, w.getDuccId(), " saving " + type);
 
 ByteArrayOutputStream baos = new ByteArrayOutputStream();

svn commit: r1714697 - in /uima/sandbox/uima-ducc/trunk/src/main: admin/db_create admin/ducc.py resources/cassandra-env.sh resources/default.ducc.properties

[challngr]

Author: challngr
Date: Mon Nov 16 22:32:39 2015
New Revision: 1714697

URL: http://svn.apache.org/viewvc?rev=1714697=rev
Log:
UIMA-4577 Allow jmx port, xmx, and 'new' memory to be specified via 
ducc.properties.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/db_create
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra-env.sh
uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/db_create
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_create?rev=1714697=1714696=1714697=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db_create (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_create Mon Nov 16 22:32:39 
2015
@@ -101,6 +101,10 @@ class DbCreate(DuccUtil):
 site_props_name = self.DUCC_HOME + 
'/resources/site.ducc.properties'
 ducc_site_properties.load(site_props_name)
 ducc_site_properties.put('ducc.database.host', db_node);
+ducc_site_properties.put('ducc.service.persistence.impl', 
'org.apache.uima.ducc.database.StateServicesDb'   , ['# Service manager 
persistence'])
+ducc_site_properties.put('ducc.job.history.impl', 
'org.apache.uima.ducc.database.HistoryManagerDb'  , ['# History and 
checkpoint'])
+ducc_site_properties.put('ducc.rm.persistence.impl' , 
'org.apache.uima.ducc.database.RmStatePersistence', ['# RM state persistence'])
+
 # if we don't die before this we need to enable db in 
site.ducc.properties and set the
 # db password into resources.private/ducc.private.properties
 ducc_site_properties.write(site_props_name)
@@ -108,6 +112,7 @@ class DbCreate(DuccUtil):
 private_props_name = self.DUCC_HOME + 
'/resources.private/ducc.private.properties'
 private_properties = Properties()
 private_properties.load(private_props_name)
+private_properties.delete('db_password')
 private_properties.put('db_password', self.database_pw, ['#Db 
password']);
 private_properties.write(private_props_name)
 

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1714697=1714696=1714697=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Mon Nov 16 22:32:39 2015
@@ -49,6 +49,22 @@ class Ducc(DuccUtil):
 return
 
 here = os.getcwd()
+
+xmx = self.ducc_properties.get('ducc.database.mem.heap')
+new = self.ducc_properties.get('ducc.database.mem.new')
+
+if ( not ( xmx == None and new == None ) ):   # if either is set
+if ( xmx == None or new == None ) :  # then both must be set
+print "Database properties ducc.database.mem.heap and 
ducc.database.mem.new must both be set."
+print 'NOTOK'
+return
+os.environ['MAX_HEAP_SIZE'] = xmx
+os.environ['HEAP_NEWSIZE'] = new
+
+jmxport = self.ducc_properties.get('ducc.database.jmx.port')
+if ( jmxport != None ):
+os.environ['JMX_PORT'] = jmxport
+
 os.chdir(self.DUCC_HOME + "/cassandra-server")
 pidfile = self.DUCC_HOME + '/state/cassandra.pid'
 CMD = "bin/cassandra -p "+  pidfile + " > /dev/null 2>&1"

Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra-env.sh
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra-env.sh?rev=1714697=1714696=1714697=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra-env.sh (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra-env.sh Mon Nov 16 
22:32:39 2015
@@ -162,10 +162,15 @@ then
 export MALLOC_ARENA_MAX=4
 fi
 
+
 # Specifies the default port over which Cassandra will be available for
 # JMX connections.
 # For security reasons, you should not expose this port to the internet.  
Firewall it if needed.
-JMX_PORT="7199"
+# DUCC: this allows the port to be set externally via ducc.properties
+if [ "x$JMX_PORT" = "x" ]
+then
+JMX_PORT="7199"
+fi
 
 
 # Here we create the arguments that will get passed to the jvm when

Modified: 
uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties?rev=1714697=1714696=1714697=diff

svn commit: r1714656 - /uima/sandbox/uima-ducc/trunk/src/main/admin/db_create

[challngr]

Author: challngr
Date: Mon Nov 16 20:30:07 2015
New Revision: 1714656

URL: http://svn.apache.org/viewvc?rev=1714656=rev
Log:
UIMA-4577 Fix typo.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/db_create

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/db_create
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_create?rev=1714656=1714655=1714656=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db_create (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_create Mon Nov 16 20:30:07 
2015
@@ -60,7 +60,7 @@ class DbCreate(DuccUtil):
 print "If no options prompts are given for expected 
parameters."
 print ""
 print "Options:"
-print "   [-, --db-password] "
+print "   [-d, --db-password] "
 print "This is the password DUCC uses to manage the database."
 print ""
 print "   [-h, -? --help]"

svn commit: r1714252 - in /uima/sandbox/uima-ducc/trunk/src/main: admin/check_ducc admin/db_loader admin/db_util.py admin/ducc_post_install admin/ducc_util.py admin/stop_ducc resources/cassandra.yaml

[challngr]

Author: challngr
Date: Fri Nov 13 18:54:15 2015
New Revision: 1714252

URL: http://svn.apache.org/viewvc?rev=1714252=rev
Log:
UIMA-4577 Scripting and config updates.

Removed:
uima/sandbox/uima-ducc/trunk/src/main/resources/database.log.config
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader
uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc
uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1714252=1714251=1714252=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Fri Nov 13 18:54:15 
2015
@@ -350,7 +350,10 @@ class CheckDucc(DuccUtil):
 self.threadpool.quit()
 
 if ( self.kill_signal != None ):
+print 'Stopping broker'
 self.stop_broker()
+print 'Stopping database'
+self.db_stop()
 
 if ( len(self.pids) == 0):
 if ( os.path.exists(self.pid_file) ):

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader?rev=1714252=1714251=1714252=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader Fri Nov 13 18:54:15 
2015
@@ -55,7 +55,7 @@ class DbLoader(DuccUtil):
 print ' Specifies the DUCC_HOME of the DUCC runtime to be 
moved into the database.'
 print '-o dburl'
 print ' Specifies the database URL to contact to store the 
items.'
-print '--noarchive'
+print '--no-archive'
 print ' Suppresses archival of the input files.'
 print ''
 print 'Notes:'

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py?rev=1714252=1714251=1714252=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py Fri Nov 13 18:54:15 
2015
@@ -26,6 +26,24 @@ def manual_config(DUCC_HOME, DUCC_HEAD):
 print ''
 print 'Note that one occurance of DUCC_HEAD will be quoted: you must 
preserve these quotes, e.g. as "' + DUCC_HEAD + '".'
 
+def update_cassandra_config(DUCC_HOME, DUCC_HEAD):
+# Read cassandra.yaml and change the things necessary to configure it 
correctly
+config = DUCC_HOME + '/cassandra-server/conf/cassandra.yaml'
+f = open(config)
+lines = []
+for line in f:
+if ( line.startswith('listen_address:') ):
+line = line.strip();
+print 'Database host is configured at', line
+if ( not DUCC_HEAD in line ):
+print 'Must reconfigure listen_address to', DUCC_HEAD
+parts = line.strip().split(':')
+old = parts[1].strip()
+ch_head = "sed -i.bak s'/" + old + "/" + DUCC_HEAD + "'/ " + 
config
+os.system(ch_head)
+
+
+
 def configure_database(DUCC_HOME, DUCC_HEAD, java, db_pw):
 # for cassandra:
 # in ducc_runtime/cassandra-server/conf we need to update cassandra.yaml 
to establish
@@ -50,22 +68,7 @@ def configure_database(DUCC_HOME, DUCC_H
 print 'Database support will be bypassed'
 return True
 
-config =  DUCC_HOME + '/cassandra-server/conf/cassandra.yaml'
-esc_home = DUCC_HOME.replace("/", "\/")# for sed
-
-# must configure the database node to be the same as the ducc head, and 
the database location
-# to be DUCC_HEAD
-ch_head = "sed -i.bak s'/DUCC_HEAD/" + DUCC_HEAD + "'/ " + config
-if ( execute(ch_head) != 0 ):
-print 'Could not configure', config + '.  You may need to recover it 
from', config+'.bak.'
-manual_config(DUCC_HOME, DUCC_HEAD)
-return False
-
-ch_home = "sed -i.bak s'/DUCC_HOME/" + esc_home + "'/ " + config
-if ( execute(ch_home) != 0):
-print 'Could not configure', config + '.  You may need to recover it 
from', config+'.bak.'
-manual_config(DUCC_HOME, DUCC_HEAD)
-return False
+update_cassandra_config(DUCC_HOME, DUCC_HEAD)
 
 here = os.getcwd()
 

svn commit: r1713863 - /uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java

[challngr]

Author: challngr
Date: Wed Nov 11 14:58:21 2015
New Revision: 1713863

URL: http://svn.apache.org/viewvc?rev=1713863=rev
Log:
UIMA-4577 Redirect i/o from stop into dev null.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java?rev=1713863=1713862=1713863=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java
 Wed Nov 11 14:58:21 2015
@@ -19,7 +19,7 @@
 package org.apache.uima.ducc.sm;
 
 import java.io.BufferedReader;
-import java.io.IOException;
+import java.io.File;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.ArrayList;
@@ -357,28 +357,15 @@ class ServiceInstance
 Map<String, String> env = pb.environment();
 env.put("DUCC_HOME", System.getProperty("DUCC_HOME"));
 
-StdioListener sin_listener = null;
-StdioListener ser_listener = null;
+pb.redirectOutput(new File("/dev/null"));
+pb.redirectError(new File("/dev/null"));
 
 int rc = 0;
 try {
 Process p = pb.start();
-
-InputStream stdout = p.getInputStream();
-   InputStream stderr = p.getErrorStream();
-
-sin_listener = new StdioListener(1, stdout, true);
-ser_listener = new StdioListener(2, stderr, true);
-Thread sol = new Thread(sin_listener);
-Thread sel = new Thread(ser_listener);
-sol.start();
-sel.start();

 rc = p.waitFor();
 logger.info(methodName, sset.getId(), "DuccServiceCancel returns 
with rc", rc);
-
-sin_listener.stop();
-ser_listener.stop();
 } catch (Throwable t) {
 logger.error(methodName, null, t);
 }

svn commit: r1713912 - in /uima/sandbox/uima-ducc/trunk/src/main/admin: db_create db_util.py ducc_post_install

[challngr]

Author: challngr
Date: Wed Nov 11 18:45:13 2015
New Revision: 1713912

URL: http://svn.apache.org/viewvc?rev=1713912=rev
Log:
UIMA-4577 Updates to scripting.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/db_create
uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/db_create
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_create?rev=1713912=1713911=1713912=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db_create (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_create Wed Nov 11 18:45:13 
2015
@@ -94,6 +94,9 @@ class DbCreate(DuccUtil):
 # configure the database for local system and initialize the schema
 db_node = self.ducc_properties.get("ducc.head")
 if ( dbu.configure_database(self.DUCC_HOME, db_node, self.jvm, 
self.database_pw)):
+private_props_name = self.DUCC_HOME + 
'/resources.private/ducc.private.properties'
+
+print 'Writing database password to', private_props_name
 ducc_site_properties = Properties();
 site_props_name = self.DUCC_HOME + 
'/resources/site.ducc.properties'
 ducc_site_properties.load(site_props_name)

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py?rev=1713912=1713911=1713912=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py Wed Nov 11 18:45:13 
2015
@@ -70,6 +70,7 @@ def configure_database(DUCC_HOME, DUCC_H
 here = os.getcwd()
 os.chdir(DUCC_HOME + "/cassandra-server")
 pidfile = DUCC_HOME + '/state/cassandra.pid'
+print 'Starting the database.  This might take a few moments if it is the 
first time.'
 CMD = "bin/cassandra -p "+  pidfile + " > /dev/null 2>&1";
 os.system(CMD);
 print "Database is started.  Waiting for initialization";

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install?rev=1713912=1713911=1713912=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install Wed Nov 11 
18:45:13 2015
@@ -60,7 +60,7 @@ class PostInstall():
 print "   [-j, --jvm] "
 print "This is the full path to java command to be used to 
start DUCC; e.g., /usr/bin/java"
 print ""
-print "   [-, --db-password] "
+print "   [-d, --db-password] "
 print "This is the password DUCC uses to manage the database."
 print ""
 print "   [-h, -? --help]"
@@ -105,24 +105,30 @@ class PostInstall():
 print 'Database is already defined.  Not configuring'
 return
 
-db_pw = raw_input("Enter database password OR 'bypass' to disable 
database support:")
-if ( db_pw == '' ):
-print "Must enter a DB password to continue."
-sys.exit(1);
+if ( self.database_pw == None ):
+db_pw = raw_input("Enter database password OR 'bypass' to disable 
database support:")
+if ( db_pw == '' ):
+print "Must enter a DB password to continue."
+sys.exit(1);
+else:
+db_pw = self.database_pw
 
 if ( db_pw == 'bypass' ):
 print 'Database support will be disabled'
 self.update_property('ducc.database.host', '--disabled--', '# 
Database support is disabled')
 return;
-else:
-self.update_property('ducc.database.host', self.ducc_head, '# 
Database location')
 
-dbu.configure_database(self.DUCC_HOME, self.ducc_head, 
self.path_to_java, db_pw)
-private_props_name = self.DUCC_HOME + 
'/resources.private/ducc.private.properties'
-private_properties = Properties()
-private_properties.load(private_props_name)
-private_properties.put('db_password', self.database_pw, ['#Db 
password']);
-private_properties.write(private_props_name)
+if ( dbu.configure_database(self.DUCC_HOME, self.ducc_head, 
self.path_to_java, db_pw) ):
+print 'Configuring DUCC to use the database.'
+
+self.update_property('ducc.database.host', self.ducc_head, '# 
Database location')
+   

svn commit: r1713913 - /uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py

[challngr]

Author: challngr
Date: Wed Nov 11 18:52:27 2015
New Revision: 1713913

URL: http://svn.apache.org/viewvc?rev=1713913=rev
Log:
UIMA-4577 Updates to scripting.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1713913=1713912=1713913=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Wed Nov 11 
18:52:27 2015
@@ -162,7 +162,6 @@ class DuccUtil(DuccBase):
 
 def db_configure(self):
 dbhost = self.ducc_properties.get('ducc.database.host')
-print '- dbhost', dbhost
 if ( dbhost == self.db_disabled ):
 self.db_bypass = True
 return;

svn commit: r1713395 - in /uima/sandbox/uima-ducc/trunk: src/main/admin/ uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/ uima-ducc-common/src/main/java/org/apache/uima/ducc/com

[challngr]

Author: challngr
Date: Mon Nov  9 11:34:50 2015
New Revision: 1713395

URL: http://svn.apache.org/viewvc?rev=1713395=rev
Log:
UIMA-4755 Make the RM node status availd to applications.

Added:

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IDbProperty.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/RmPersistenceFactory.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbAlive.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java

uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1713395=1713394=1713395=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Mon Nov  9 
11:34:50 2015
@@ -162,6 +162,7 @@ class DuccUtil(DuccBase):
 
 def db_configure(self):
 dbhost = self.ducc_properties.get('ducc.database.host')
+print '- dbhost', dbhost
 if ( dbhost == self.db_disabled ):
 self.db_bypass = True
 return;
@@ -189,12 +190,12 @@ class DuccUtil(DuccBase):
 if ( self.system == 'Darwin'):
 ps = 'ps -eo user,pid,comm,args ' + pid
 else:
-ps = 'ps -eo user:14,pid,comm,args ' + pkd
+ps = 'ps -eo user:14,pid,comm,args ' + pid
 lines = self.popen(ps)
 
 for line in lines:
 line = line.strip()
-if (pid in line):
+if (pid in line and 'cassandra' in line):
 return True
 return False
 
@@ -242,7 +243,7 @@ class DuccUtil(DuccBase):
 line = lines.readline().strip()
 except:
 break
-# print '[]', line
+#print '[]', line
 
 if ( not line ):
 break

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IDbProperty.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IDbProperty.java?rev=1713395=1713394=1713395=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IDbProperty.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IDbProperty.java
 Mon Nov  9 11:34:50 2015
@@ -39,8 +39,9 @@ public interface IDbProperty
  // than does SQL, we define a 
translation from the "properties" key to
  // legal SQL syntactic names.  DB 
does not translate, user must provide
  // a suitable translation.
+boolean isIndex();// If true, create a secondary index.
 
-// If we update this we may have to update db methods that use it
+// If we update this we may have to update db methods that use it.  We'll 
try to avoid collections for now.
 public enum Type {
 String,// Java String
 Blob,  // Java seriali

svn commit: r1712850 - in /uima/sandbox/uima-ducc/trunk: src/main/admin/ src/main/assembly/ src/main/resources/ src/main/resources/private/ uima-ducc-common/src/main/java/org/apache/uima/ducc/common/p

[challngr]

Author: challngr
Date: Thu Nov  5 19:23:05 2015
New Revision: 1712850

URL: http://svn.apache.org/viewvc?rev=1712850=rev
Log:
UIMA-4755 Scripting and build updates.

Added:
uima/sandbox/uima-ducc/trunk/src/main/admin/db_create   (with props)
uima/sandbox/uima-ducc/trunk/src/main/admin/db_util.py
uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml   (with 
props)
Removed:
uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml.prep
uima/sandbox/uima-ducc/trunk/src/main/resources/private/database.xml
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml
uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra-env.sh

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbAlive.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1712850=1712849=1712850=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Thu Nov  5 19:23:05 
2015
@@ -55,7 +55,7 @@ class CheckDucc(DuccUtil):
 if ( self.db_bypass == True ):
 return True
 
-ret = self.db_alive()
+ret = self.db_alive(3)
 if ( ret ):
 print 'The database is running'
 else:

Added: uima/sandbox/uima-ducc/trunk/src/main/admin/db_create
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_create?rev=1712850=auto
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db_create (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_create Thu Nov  5 19:23:05 
2015
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# ---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ---
+
+
+import os
+import sys
+import getopt
+
+import shutil
+import subprocess
+from  stat import *
+
+from ducc_util import DuccUtil
+
+
+from ducc_base import Properties
+from ducc_base import Property
+
+from ducc_base import find_ducc_home
+from ducc_base import find_localhost
+
+from ducc import Ducc
+
+import db_util as dbu
+
+# 
+# Create and initialize the DUCC database.  In a new installation this is 
handled by
+# ducc_post_install.  Older installations need to run db_create as one of the 
steps
+# of migration to the DB.
+#
+class DbCreate(DuccUtil):
+
+def usage(self, msg):
+
+if ( msg != None ):
+print ' '.join(msg)
+
+  
+print 'DbCreate configures the database and installs the schema.'
+print ''
+print "Usage:"
+print "   db_create [options]"
+print "If no options prompts are given for expected 
parameters."
+print ""
+print "Options:"
+print "   [-n, --head-node] "
+print "This is the name of the host that will run the DUCC 
management processes."
+print ""
+print "   [-, --db-p

svn commit: r1712117 - /uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java

[challngr]

Author: challngr
Date: Mon Nov  2 19:15:07 2015
New Revision: 1712117

URL: http://svn.apache.org/viewvc?rev=1712117=rev
Log:
UIMA-4587 Stop updating meta state after unregister.  Happens if state 
continues to arrive after unregister.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java?rev=1712117=1712116=1712117=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
 Mon Nov  2 19:15:07 2015
@@ -842,8 +842,10 @@ public class ServiceSet
throws Exception
 {
// String methodName = "saveMetaProperties";
-// UIMA-4587 Why bypass, as state can still dribble in.
-// if ( isDeregistered() ) return;
+if ( isDeregistered() ) return;// we may have deleted 
the properties but stuff
+   // lingers out of our 
control.  no more updates
+   // which can leave junk 
in the registry directory
+   // for file-based 
registry.
 
 prepareMetaProperties();
 stateHandler.updateMetaProperties(id, meta_props);

svn commit: r1712113 - in /uima/sandbox/uima-ducc/trunk: uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java uima-ducc-sm/src/main/java/org/apache/uima/d

[challngr]

Author: challngr
Date: Mon Nov  2 19:07:56 2015
New Revision: 1712113

URL: http://svn.apache.org/viewvc?rev=1712113=rev
Log:
UIMA-4587 Add sync to properties store.  Adjust ping-only ServiceInstance 
start() signature.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java

uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingOnlyServiceInstance.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java?rev=1712113=1712112=1712113=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java
 Mon Nov  2 19:07:56 2015
@@ -149,7 +149,7 @@ public class StateServices implements IS
}
 
 // Try to write properties file, using a temp file as backup in case it 
fails.
-private boolean writeProperties(DuccId id, Properties props, File pfile, 
File pfile_tmp, String type)
+private synchronized boolean writeProperties(DuccId id, Properties props, 
File pfile, File pfile_tmp, String type)
 {

String methodName = "saveProperties";

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingOnlyServiceInstance.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingOnlyServiceInstance.java?rev=1712113=1712112=1712113=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingOnlyServiceInstance.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingOnlyServiceInstance.java
 Mon Nov  2 19:07:56 2015
@@ -46,7 +46,7 @@ class PingOnlyServiceInstance
 // this.state = dwj.getJobState();
 // }
 
-long start(String spec, DuccProperties meta_props)
+long start(DuccProperties svc_props, DuccProperties meta_props)
 {
String methodName = "start";
 

svn commit: r1712133 - /uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install

[challngr]

Author: challngr
Date: Mon Nov  2 20:03:01 2015
New Revision: 1712133

URL: http://svn.apache.org/viewvc?rev=1712133=rev
Log:
UIMA-4577 Remove obsolete 'database' module.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install?rev=1712133=1712132=1712133=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install Mon Nov  2 
20:03:01 2015
@@ -37,8 +37,6 @@ from ducc_base import which
 
 from ducc import Ducc
 
-import database as db
-
 class PostInstall():
 
 def usage(self, msg):

svn commit: r1712120 - in /uima/sandbox/uima-ducc/trunk/src/main/admin: database.py db.py db_console

[challngr]

Author: challngr
Date: Mon Nov  2 19:20:33 2015
New Revision: 1712120

URL: http://svn.apache.org/viewvc?rev=1712120=rev
Log:
UIMA-4577 Remove unused code.

Removed:
uima/sandbox/uima-ducc/trunk/src/main/admin/database.py
uima/sandbox/uima-ducc/trunk/src/main/admin/db.py
uima/sandbox/uima-ducc/trunk/src/main/admin/db_console

svn commit: r1711768 [1/2] - in /uima/sandbox/uima-ducc/trunk: ./ src/main/admin/ src/main/assembly/ src/main/resources/ uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ uima-ducc-examp

[challngr]

Author: challngr
Date: Sun Nov  1 14:09:47 2015
New Revision: 1711768

URL: http://svn.apache.org/viewvc?rev=1711768=rev
Log:
UIMA-4577 Cassandra db support; packaging and scriptng.

Added:
uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra-env.sh   (with 
props)
uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml.prep   (with 
props)

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbAlive.java
Modified:
uima/sandbox/uima-ducc/trunk/pom.xml
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc
uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml
uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties
uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/stop_sim
uima/sandbox/uima-ducc/trunk/uima-ducc-parent/pom.xml

Modified: uima/sandbox/uima-ducc/trunk/pom.xml
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/pom.xml?rev=1711768=1711767=1711768=diff
==
--- uima/sandbox/uima-ducc/trunk/pom.xml (original)
+++ uima/sandbox/uima-ducc/trunk/pom.xml Sun Nov  1 14:09:47 2015
@@ -336,6 +336,27 @@
   
 
 
+
+  
+  unpack cassandra Bin
+  prepare-package
+  unpack
+  
+true
+
+  
+org.apache.cassandra
+apache-cassandra
+${cassandra.server.version}
+tar.gz
+bin
+
${project.build.directory}/cassandra
+  
+
+  
+
+
 



Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1711768=1711767=1711768=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Sun Nov  1 14:09:47 2015
@@ -27,7 +27,6 @@ import time
 
 from ducc_util import DuccUtil
 from local_hooks import verify_slave_node
-import database as db
 
 from properties import Properties
 from properties import Property
@@ -35,50 +34,43 @@ from properties import Property
 class Ducc(DuccUtil):
 
 
-def run_db(self, background):
-# The dom gets loaded by the import above - or not
-if ( not db.domloaded ):
-print "Unable to read database configuration; insure the installed 
Python supports xml.dom.minidom"
-print "Note that Python must be at least version 2.6 but not 3.x.  
You are running version", sys.version_info
-return
-
-print '(re)Starting the database'
-
-# first write this out so it's easy to find in java without having to 
parse xml
-(jvm_parms, classpath, db_rt, dburl, dbrest, dbroot) = self.db_parms
-db_pw_file = Properties()
-comment = []
-comment.append('# Do not modify this file, it is auto-generated.')
-comment.append('# This is extracted from the database configuration 
and written to this file on every boot of the db')
-p = Property('db_password', dbroot, comment)
-db_pw_file.put_property(p)
-db_pw_filename = self.DUCC_HOME + '/resources.private/db_password'
-db_pw_file.write(db_pw_filename)
-os.chmod(db_pw_filename, 0700)
+def run_db(self):
 
-main = 'com.orientechnologies.orient.server.OServerMain'
-
-jp = ''
-for k in jvm_parms.keys():
-v = jvm_parms[k]
-if ( v == None ):
-jp = jp + k + ' '
-else:
-jp = jp + k + '=' + v + ' '
-
-
-if ( self.db_jvm_args != None ):
-jp = jp + ' ' + self.db_jvm

svn commit: r1711768 [2/2] - in /uima/sandbox/uima-ducc/trunk: ./ src/main/admin/ src/main/assembly/ src/main/resources/ uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ uima-ducc-examp

[challngr]

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java?rev=1711768=1711767=1711768=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
 Sun Nov  1 14:09:47 2015
@@ -19,36 +19,43 @@
 
 package org.apache.uima.ducc.database;
 
+import java.io.FileOutputStream;
 import java.util.List;
+import java.util.Properties;
+import java.util.UUID;
 
 import org.apache.uima.ducc.common.utils.DuccLogger;
 
+import com.datastax.driver.core.AuthProvider;
 import com.datastax.driver.core.Cluster;
 import com.datastax.driver.core.Host;
 import com.datastax.driver.core.Metadata;
+import com.datastax.driver.core.PlainTextAuthProvider;
 import com.datastax.driver.core.Session;
 import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.exceptions.AuthenticationException;
 
 public class DbCreate
 {
+static final String DUCC_KEYSPACE = "ducc";
+static final String PASSWORD_KEY  = "db_password";
+static final String PASSWORD_FILE = "database.password";
+
 DuccLogger logger = null;
 String dburl;
-String adminid = "root";
+String adminid = null;
 String adminpw = null;
 
 private Cluster cluster;
 private Session session;
 
-public DbCreate(String dburl)
-{
-this.dburl = dburl;
-}
 
-
-public DbCreate(String dburl, DuccLogger logger)
+public DbCreate(String dburl, DuccLogger logger, String adminid, String 
adminpw)
 {
 this.dburl = dburl;
 this.logger = logger;
+this.adminid = adminid;
+this.adminpw = adminpw;
 }
 
 public DbCreate(String dburl, String adminid, String adminpw)
@@ -59,11 +66,55 @@ public class DbCreate
 }
 
 public void connect()
+throws Exception
 {
 String methodName = "connect";
-cluster = Cluster.builder()
-.addContactPoint(dburl)
-.build();
+
+String dh = System.getProperty("DUCC_HOME");
+if ( dh == null ) {
+throw new IllegalArgumentException("DUCC_HOME must be set as a 
system property: -DDUCC_HOME=whatever");
+}
+
+try {
+// If we're here, we must first of all get rid of the cassandra su 
and set up our own
+
+AuthProvider auth = new PlainTextAuthProvider("cassandra", 
"cassandra");
+cluster = Cluster.builder()
+.withAuthProvider(auth)
+.addContactPoint(dburl)
+.build();
+
+session = cluster.connect();
+session.execute("CREATE USER IF NOT EXISTS " + adminid + " with 
password '" + adminpw + "' SUPERUSER");
+cluster.close();
+doLog(methodName, "Created user " + adminid);
+
+Properties props = new Properties();
+props.setProperty(PASSWORD_KEY, adminpw);
+FileOutputStream fos = new FileOutputStream(dh + 
"/resources.private/" + PASSWORD_FILE);
+props.store(fos, "Db private configuration");
+fos.close();
+
+auth = new PlainTextAuthProvider(adminid, adminpw);
+cluster = Cluster.builder()
+.withAuthProvider(auth)
+.addContactPoint(dburl)
+.build();
+session = cluster.connect();
+   
+String uglypw = UUID.randomUUID().toString();
+session.execute("ALTER USER cassandra  with password '" + uglypw + 
"' NOSUPERUSER");
+doLog(methodName, "Changed default super user's password and 
revoked its superuser authority.");
+doLog(methodName, "From this point, this DB can only be accessed 
in super user mode by user 'ducc'");
+
+} catch (AuthenticationException e ) {
+// if we get here the default super user isn't working and we 
expect a valid id and password
+AuthProvider auth = new PlainTextAuthProvider(adminid, adminpw);
+cluster = Cluster.builder()
+.withAuthProvider(auth)
+.addContactPoint(dburl)
+.build();
+}
 
 Metadata metadata = cluster.getMetadata();
 doLog(methodName, "Connected to cluster: %s\n", 
metadata.getClusterName());
@@ -186,14 +237,14 @@ public class DbCreate
 
 public static void main(String[] args)
 {
-if ( args.length != 1 ) {
-System.out.println("Usage: DbCreate ");
+if ( args.length != 3 ) {
+System.out.println("Usage: DbCreate database_url db_id db_pw");
 System.exit(1);
 }
 
 DbCreate 

svn commit: r1711846 - in /uima/sandbox/uima-ducc/trunk/src/main/admin: check_ducc ducc_util.py

[challngr]

Author: challngr
Date: Sun Nov  1 19:39:12 2015
New Revision: 1711846

URL: http://svn.apache.org/viewvc?rev=1711846=rev
Log:
UIMA-4577 Db updates.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1711846=1711845=1711846=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Sun Nov  1 19:39:12 
2015
@@ -52,7 +52,7 @@ class CheckDucc(DuccUtil):
 return
 
 def verify_database(self):
-if ( self.db_parms == self.db_disabled ):
+if ( self.db_bypass == True ):
 return True
 
 ret = self.db_alive()

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1711846=1711845=1711846=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Sun Nov  1 
19:39:12 2015
@@ -163,9 +163,10 @@ class DuccUtil(DuccBase):
 def db_configure(self):
 dbhost = self.ducc_properties.get('ducc.database.host')
 if ( dbhost == self.db_disabled ):
-self.db_disabled = True
+self.db_bypass = True
+return;
 else:
-self.db_disabled = False
+self.db_bypass = False
 
 dbprops = Properties()
 dbprops.load(self.DUCC_HOME + '/resources.private/database.password')
@@ -196,7 +197,7 @@ class DuccUtil(DuccBase):
 
 # contact the database and see how useful it seems to be
 def db_alive(self):
-if ( self.db_disabled == True ):
+if ( self.db_bypass == True ):
 return True
 
 dbnode = self.ducc_properties.get('ducc.state.database.url')
@@ -219,7 +220,7 @@ class DuccUtil(DuccBase):
 def db_start(self):
 
 # bypass all of this for the initial delivery
-if ( self.db_disabled == True) :
+if ( self.db_bypass == True) :
 print '   (Bypass database start because ducc.database.host =', 
self.db_disabled + ')'
 return True
 
@@ -259,8 +260,8 @@ class DuccUtil(DuccBase):
 
 def db_stop(self):
 
-if ( self.db_disabled == True) :
-print '   (Bypass database start because ducc.database.host =', 
self.db_disabled + ')'
+if ( self.db_bypass == True) :
+print '   (Bypass database stop because ducc.database.host =', 
self.db_disabled + ')'
 return True
 
 pidfile = self.DUCC_HOME + '/state/cassandra.pid'

svn commit: r1711085 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc: DbQuery.java Server.java Shutdown.java SmLoader.java

[challngr]

Author: challngr
Date: Wed Oct 28 17:59:03 2015
New Revision: 1711085

URL: http://svn.apache.org/viewvc?rev=1711085=rev
Log:
UIMA-4577 Switch to Cassandra, drop Orient.

Removed:

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/DbQuery.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/Server.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/Shutdown.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/SmLoader.java

svn commit: r1711088 [4/4] - in /uima/sandbox/uima-ducc/trunk: src/main/admin/ src/main/assembly/ src/main/resources/ uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/ uima-ducc-

[challngr]

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java?rev=1711088=1711087=1711088=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
 Wed Oct 28 18:12:53 2015
@@ -19,20 +19,23 @@
 package org.apache.uima.ducc.database;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.UUID;
 
+import org.apache.uima.ducc.common.persistence.IDbProperty;
 import org.apache.uima.ducc.common.persistence.services.IStateServices;
 import org.apache.uima.ducc.common.persistence.services.StateServicesDirectory;
 import org.apache.uima.ducc.common.persistence.services.StateServicesSet;
 import org.apache.uima.ducc.common.utils.DuccLogger;
 import org.apache.uima.ducc.common.utils.DuccProperties;
 import org.apache.uima.ducc.common.utils.id.DuccId;
-import org.apache.uima.ducc.database.DbConstants.DbCategory;
-import org.apache.uima.ducc.database.DbConstants.DbVertex;
 
-import com.tinkerpop.blueprints.impls.orient.OrientVertex;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.SimpleStatement;
 
 public class StateServicesDb
 implements IStateServices
@@ -40,25 +43,45 @@ public class StateServicesDb
private DuccLogger logger = null;
 private DbManager dbManager;
 
+private static final String SVC_TABLE = "ducc." + 
SvcRegProps.TABLE_NAME.pname();
+private static final String META_TABLE = "ducc." + 
SvcMetaProps.TABLE_NAME.pname();
+// (My) db conventions are that everything must follow the conventions of 
IDbProperty.  SM
+// uses properties directly.  Maybe we'll change this some time.  For now, 
we need to efficiently
+// convert a Properties object into a Map keyed on IDbProperty; hence 
these two convenient maps
+// from string to property.
+Map s2regProps = new HashMap();
+Map s2metaProps = new HashMap();
 public StateServicesDb()
 {
+for ( SvcRegProps p : SvcRegProps.values() ) {
+s2regProps.put(p.pname(), p);
+}
+for ( SvcMetaProps p : SvcMetaProps.values() ) {
+s2metaProps.put(p.pname(), p);
+}
 }
 
-private boolean init(String dburl)
+private boolean init(String dburl, DbManager dbm)
 throws Exception
 {
+   String methodName = "init";
 // log4j issue - the caller must pass in a logger configured correctly 
for the component
 //   to which it belongs or it won't get sent to the right 
appender.
 
-boolean ret = false;
-try {
-dbManager = new DbManager(dburl, logger);
-dbManager.init();
-ret = true;
-} catch ( Exception e ) {
-// logger.error("StateServicesDb", null, "Cannot open the service 
database:", e);
-throw e;
+boolean ret = true;
+if ( dbm != null ) {
+this.dbManager = dbm;
+} else {
+try {
+dbManager = new DbManager(dburl, logger);
+dbManager.init();
+ret = true;
+} catch ( Exception e ) {
+logger.error(methodName, null, "Cannot open the state 
database:", e);
+throw e;
+}
 }
+
 return ret;
 }
 
@@ -67,60 +90,59 @@ public class StateServicesDb
 {
this.logger = logger;
 String stateUrl = System.getProperty("ducc.state.database.url");
-return init(stateUrl);
+return init(stateUrl, null);
 }
 
-/** 
- * Helper for restoring service registrations.  This looks in the 
non-history part of the DB.
- */
-private List getListOfType(DbVertex type)
+// package only, for the loader
+boolean init(DuccLogger logger, DbManager dbManager)
+   throws Exception
 {
-String methodName = "getSvcList";
-
-if ( dbManager == null ) {
-logger.error(methodName, null, "Service database is not 
initialized.");
-return new ArrayList();// avoid NPE in caller
-}
-
-List ret = null;
-DbHandle h = null;
-   try {
-   h = dbManager.open();
-ret = h.listObjectsOfType(type, DbCategory.SmReg);
-   } catch (Throwable e) {
-logger.error(methodName, null, e);
-   } finally {
-  

svn commit: r1711089 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database: DbConstants.java DbListLoader.java

[challngr]

Author: challngr
Date: Wed Oct 28 18:17:40 2015
New Revision: 1711089

URL: http://svn.apache.org/viewvc?rev=1711089=rev
Log:
UIMA-4577 Switch from OrientDb to Cassandra.

Removed:

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbConstants.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbListLoader.java

svn commit: r1711092 - /uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java

[challngr]

Author: challngr
Date: Wed Oct 28 18:20:05 2015
New Revision: 1711092

URL: http://svn.apache.org/viewvc?rev=1711092=rev
Log:
UIMA-4577 Switch from OrientDb to Cassandra.

Removed:

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java

svn commit: r1711091 - /uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbObject.java

[challngr]

Author: challngr
Date: Wed Oct 28 18:18:59 2015
New Revision: 1711091

URL: http://svn.apache.org/viewvc?rev=1711091=rev
Log:
UIMA-4577 Switch from OrientDb to Cassandra.

Removed:

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbObject.java

svn commit: r1711088 [3/4] - in /uima/sandbox/uima-ducc/trunk: src/main/admin/ src/main/assembly/ src/main/resources/ uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/ uima-ducc-

[challngr]

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java?rev=1711088=1711087=1711088=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
 Wed Oct 28 18:12:53 2015
@@ -20,180 +20,134 @@
 package org.apache.uima.ducc.database;
 
 import java.io.FileInputStream;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.Properties;
 
 import org.apache.uima.ducc.common.utils.DuccLogger;
-import org.apache.uima.ducc.database.DbConstants.DbEdge;
-import org.apache.uima.ducc.database.DbConstants.DbVertex;
-
-import com.orientechnologies.orient.client.remote.OServerAdmin;
-import com.orientechnologies.orient.core.intent.OIntent;
-import com.tinkerpop.blueprints.impls.orient.OrientGraph;
-import com.tinkerpop.blueprints.impls.orient.OrientGraphFactory;
-import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx;
 
+import com.datastax.driver.core.BoundStatement;
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.Host;
+import com.datastax.driver.core.Metadata;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.SimpleStatement;
+
+/**
+ * Provide a common point for contacting the db, acquiring sessions/handles to 
it, managing the db,
+ * closing, etc.
+ */
 public class DbManager
 {
 
 String dburl;
 DuccLogger logger;
 
-OrientGraphFactory  factory;
-
-Map name_to_vertex = new HashMap();
-Map   name_to_edge   = new HashMap();
-
-// private ODatabaseDocumentTx documentDb;
+private Cluster cluster;// only one
+private Session session;// only one - it's thread safe and 
manages a connection pool
 
 public DbManager(String dburl, DuccLogger logger)
 throws Exception
 {
 this.dburl = dburl;
 this.logger = logger;
-
-for ( DbVertex o :  DbVertex.values() ) {
-name_to_vertex.put(o.pname(), o);
-}
-for ( DbEdge o :  DbEdge.values() ) {
-name_to_edge.put(o.pname(), o);
-}
-
 }
-
-DbVertex vertexType(String v)
-{
-return name_to_vertex.get(v);
-}
-
-DbEdge EdgeType(String e)
-{
-return name_to_edge.get(e);
-}
-
 
 boolean checkForDatabase()
throws Exception
 {
 String methodName = "checkForDatabase";
-String pw = dbPassword();
-OServerAdmin admin = null;
-
-boolean ret = true;
-try {
-admin = new OServerAdmin(dburl);
-admin.connect("root", pw);   // connect to the server
-   
-if ( ! admin.existsDatabase("plocal") ) {
-logger.info(methodName, null, "Database " + dburl + " does not 
exist.");
-ret = false;
-}
-} finally {
-if ( admin != null) admin.close();
-}
-return ret;
+logger.warn(methodName, null, "Not yet implemented.");
+return true;
 }
 
 public void drop()
 throws Exception
 {
-if ( dburl.startsWith("remote") ) {
-OServerAdmin admin = null;
-try {
-String pw = dbPassword();
-   admin = new OServerAdmin(dburl);
-admin.connect("root", pw);   // connect to the 
server
-admin.dropDatabase("plocal");   
-admin.close();
-} finally {
-if ( admin != null ) admin.close();
-}
-} else  {
-OrientGraphNoTx graphDb = factory.getNoTx();// the graph 
instance
-if ( graphDb == null ) {
-throw new IllegalStateException("Cannot allocate graph 
instance for " + dburl);
-}
-graphDb.drop();
-}
+String methodName = "drop";
+logger.warn(methodName, null, "Drop is not implemented yet.");
 }
 
 public synchronized DbHandle open()
 throws Exception
 {
-   OrientGraph graphDb = factory.getTx();// the graph instance
-if ( graphDb == null ) {
-throw new IllegalStateException("Cannot allocate graph instance 
for " + dburl);
+if ( session == null ) {
+session = cluster.connect();
 }
-
-graphDb.setUseLightweightEdges(true);
-return new DbHandle(this, graphDb);
+
+  

svn commit: r1711088 [1/4] - in /uima/sandbox/uima-ducc/trunk: src/main/admin/ src/main/assembly/ src/main/resources/ uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/ uima-ducc-

[challngr]

Author: challngr
Date: Wed Oct 28 18:12:53 2015
New Revision: 1711088

URL: http://svn.apache.org/viewvc?rev=1711088=rev
Log:
UIMA-4577 Switch from OrientDb to Cassandra.

Added:

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IDbProperty.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbVerify.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ReadCkpt.java
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml
uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/RmPersistenceFactory.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/java/org/apache/uima/ducc/common/test/cmd/StateServicesTest.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/pom.xml

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java

uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCheckpoint.java
uima/sandbox/uima-ducc/trunk/uima-ducc-parent/pom.xml

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java

uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java

uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java

uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/pom.xml

uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java

uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java

uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java

uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java

uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/IServicesRegistry.java

uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/sort/ServicesSortCache.java

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1711088=1711087=1711088=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Wed Oct 28 18:12:53 2015
@@ -115,6 +115,9 @@ class Ducc(DuccUtil):
 def add_to_classpath(self, lib):
 os.environ['CLASSPATH'] = os.environ['CLASSPATH

svn commit: r1708160 - /uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py

[challngr]

Author: challngr
Date: Mon Oct 12 16:47:16 2015
New Revision: 1708160

URL: http://svn.apache.org/viewvc?rev=1708160=rev
Log:
UIMA-4577 Remove some debugging lines.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1708160=1708159=1708160=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Mon Oct 12 
16:47:16 2015
@@ -241,6 +241,7 @@ class DuccUtil(DuccBase):
 
 # bypass all of this for the initial delivery
 if ( self.db_parms == self.db_disabled ):
+print '   (Bypass database start because ducc.database.host =', 
self.db_disabled + ')'
 return True
 
 print 'Starting database'
@@ -404,7 +405,7 @@ class DuccUtil(DuccBase):
 
 def nohup(self, cmd, showpid=True):
 cmd = ' '.join(cmd)
-print ' nohup', cmd, ''
+# print ' nohup', cmd, ''
 devnw = open(os.devnull, 'w')
 devnr = open(os.devnull, 'r')
 ducc = subprocess.Popen(cmd, shell=True, stdin=devnr, stdout=devnw, 
stderr=devnw)
@@ -417,7 +418,7 @@ class DuccUtil(DuccBase):
 def ssh(self, host, do_wait, *CMD):
 
 cmd = ' '.join(CMD)
-print 'ssh -o BatchMode=yes -o ConnectTimeout=10', host, cmd
+# print 'ssh -o BatchMode=yes -o ConnectTimeout=10', host, cmd
 if ( do_wait ):
 return self.popen('ssh -q -o BatchMode=yes -o ConnectTimeout=10', 
host, cmd)
 else:

svn commit: r1708155 - /uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java

[challngr]

Author: challngr
Date: Mon Oct 12 16:24:25 2015
New Revision: 1708155

URL: http://svn.apache.org/viewvc?rev=1708155=rev
Log:
UIMA-4577 RM node state persistence.  Loader updates.  Service reg updates.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java?rev=1708155=1708154=1708155=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
 Mon Oct 12 16:24:25 2015
@@ -530,7 +530,7 @@ public class DbLoader
 try {
 
 OGlobalConfiguration.USE_WAL.setValue(false);
-OGlobalConfiguration.USE_LOG.setValue(false);
+// OGlobalConfiguration.USE_LOG.setValue(false);
 
 OGlobalConfiguration.dumpConfiguration(System.out);
 

svn commit: r1708149 [1/2] - in /uima/sandbox/uima-ducc/trunk: src/main/admin/ src/main/resources/ uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/monitor/ uima-ducc-common/src/main/java/org/

[challngr]

Author: challngr
Date: Mon Oct 12 16:10:55 2015
New Revision: 1708149

URL: http://svn.apache.org/viewvc?rev=1708149=rev
Log:
UIMA-4577 RM node state persistence.  Loader updates.  Service reg updates.

Added:

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/RmPersistenceFactory.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbListLoader.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties

uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/monitor/AgentMonitor.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/ANodeStability.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbConstants.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/SmLoader.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/NodeStability.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/event/ResourceManagerEventListener.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ISchedulerMain.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java

uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java

uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java

uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/event/ServiceManagerEventListener.java

uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1708149=1708148=1708149=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Mon Oct 12 16:10:55 2015
@@ -66,6 +66,10 @@ class Ducc(DuccUtil):
 else:
 jp = jp + k + '=' + v + ' '
 
+
+if ( self.db_jvm_args != None ):
+jp = jp + ' ' + self.db_jvm_args
+
 cmd = ' '.join(['nohup', self.java(), jp, '-cp', classpath, main, '&'])
 print cmd
 
@@ -177,6 +181,7 @@ class Ducc(DuccUtil):
 jvm_opts.append("-Dducc.rm.override.dram=" + rmoverride)
 if ( self.rm_jvm_args != None ):
 jvm_opts.append(self.rm_jvm_args)
+self.add_to_classpath(ducc_home + '/lib/orientdb/*')   
 

svn commit: r1708149 [2/2] - in /uima/sandbox/uima-ducc/trunk: src/main/admin/ src/main/resources/ uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/monitor/ uima-ducc-common/src/main/java/org/

[challngr]

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java?rev=1708149=1708148=1708149=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
 Mon Oct 12 16:10:55 2015
@@ -79,9 +79,14 @@ public class DbLoader
 String archive_key  = IStateServices.archive_key;
 String archive_flag = IStateServices.archive_flag;
 
-int nthreads = 40;
+int nthreads = 20;
 AtomicInteger counter = new AtomicInteger(0);
 
+//int joblimit = 1;
+//int reservationlimit = 1;
+//int servicelimit = 1;
+//int registrylimit= 1;
+
 int joblimit = Integer.MAX_VALUE;
 int reservationlimit = Integer.MAX_VALUE;
 int servicelimit = Integer.MAX_VALUE;
@@ -111,6 +116,13 @@ public class DbLoader
 System.exit(1);
 }
 
+jobHistory = from + jobHistory;
+reservationHistory = from + reservationHistory;
+serviceHistory = from + serviceHistory;
+serviceRegistryHistory = from + serviceRegistryHistory;
+serviceRegistry= from + serviceRegistry;
+checkpointFile = from + checkpointFile;
+
 f = new File(to);
 if ( ! f.isDirectory() ) {
 System.out.println("'to' must be a directory");
@@ -120,39 +132,26 @@ public class DbLoader
 String databasedir =  to + "/database/databases";
 String databasename = databasedir + "/DuccState";
 // We always use a non-networked version for loading
-state_url = "plocal:" + databasedir + "/DuccState";
+//state_url = "plocal:" + databasedir + "/DuccState";
+state_url = "remote:bluej538/DuccState";
 System.setProperty("ducc.state.database.url", state_url);
 
-f = new File(databasedir);
-if ( f.exists() ) {
-f = new File(databasename);
-if ( f.exists() ) {
-logger.info(methodName, null, "Dropping existing database.");
-DbManager dbm = new DbManager(state_url, logger);
-dbm.init();
-dbm.drop();
-dbm.shutdown();
-}
-} else {
-try {
-if ( ! f.mkdirs() ) {
-System.out.println("Cannot create database directory: " + 
databasedir);
+if ( state_url.startsWith("plocal") ) {
+f = new File(databasedir);
+if ( !f.exists() ) {
+try {
+if ( ! f.mkdirs() ) {
+System.out.println("Cannot create database directory: 
" + databasedir);
+System.exit(1);
+}
+System.out.println("Created database directory " + 
databasedir);
+} catch ( Exception e ) {
+System.out.println("Cannot create database directory: " + 
databasedir + ":" + e.toString());
 System.exit(1);
 }
-System.out.println("Created database directory " + 
databasedir);
-} catch ( Exception e ) {
-System.out.println("Cannot create database directory: " + 
databasedir + ":" + e.toString());
-System.exit(1);
 }
 }
 
-
-jobHistory = from + jobHistory;
-reservationHistory = from + reservationHistory;
-serviceHistory = from + serviceHistory;
-serviceRegistryHistory = from + serviceRegistryHistory;
-serviceRegistry= from + serviceRegistry;
-checkpointFile = from + checkpointFile;
 }
 
 void closeStream(InputStream in)
@@ -164,6 +163,7 @@ public class DbLoader
 {
 String methodName = "loadJobs";
 
+logger.info(methodName, null, "  Load jobs 
");
 File dir = new File(jobHistory);
 if ( !dir.isDirectory() ) {
 logger.info(methodName, null, "Cannot find job history; skipping 
load of jobs.");
@@ -237,6 +237,7 @@ public class DbLoader
 {
 String methodName = "loadReservations";
 
+logger.info(methodName, null, "  Load reservations 
");
 File dir = new File(reservationHistory);
 if ( ! dir.isDirectory() ) {
 logger.info(methodName, null, "No reservation directory found; 
skipping database load of reservations.");
@@ -310,6 +311,8 @@ public class DbLoader
 public void loadServices()
 {
 String 

svn commit: r1708161 - /uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties

[challngr]

Author: challngr
Date: Mon Oct 12 16:52:03 2015
New Revision: 1708161

URL: http://svn.apache.org/viewvc?rev=1708161=rev
Log:
UIMA-4577 Set RM persistence to null implementation.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties

Modified: 
uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties?rev=1708161=1708160=1708161=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties 
(original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties Mon 
Oct 12 16:52:03 2015
@@ -540,5 +540,8 @@ ducc.service.persistence.impl = org.apac
 #ducc.job.history.impl = org.apache.uima.ducc.database.HistoryManagerDb
 ducc.job.history.impl = 
org.apache.uima.ducc.transport.event.common.history.HistoryPersistenceManager
 
+ducc.rm.persistence.impl = 
org.apache.uima.ducc.common.persistence.rm.NullRmStatePersistence
+#ducc.rm.persistence.impl = 
org.apache.uima.ducc.database.RmStatePersistence
+
 # end database things 
---
 

svn commit: r1707115 - /uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml

[challngr]

Author: challngr
Date: Tue Oct  6 18:28:22 2015
New Revision: 1707115

URL: http://svn.apache.org/viewvc?rev=1707115=rev
Log:
UIMA-4577 Remove accidentally added logging stuff.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml

Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml?rev=1707115=1707114=1707115=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml Tue Oct  6 
18:28:22 2015
@@ -35,7 +35,7 @@

  

-
+  

  
  
@@ -50,24 +50,16 @@
  
   
   
-   
- 
- 
-   
- 
-  
-
-  

 
 
-
-
+
+
  

  

-
+  

 
 
@@ -81,8 +73,8 @@

 
 
-
-
+
+
  

  
@@ -92,6 +84,7 @@
 
 
 
+
  

  
@@ -101,15 +94,7 @@
 
 
 
- 
-   
- 
-   
-  
-   
-
-
-
+
  

  
@@ -118,8 +103,8 @@

 
 
-
-
+
+
  

  
@@ -129,6 +114,7 @@
 
 
 
+
  

  
@@ -154,14 +140,13 @@
  

   
-

  
   

 

- 
+ 
   

 
@@ -171,7 +156,7 @@

 

- 
+ 
   

 
@@ -180,28 +165,18 @@
   

 
-   
- 
-  
-   
-   
-   
- 
-  
-   
-   

- 
+ 
  

 
-   
+   
  
  

 

- 
+ 
  

 
@@ -220,64 +195,43 @@
   

 
-   
- 
-  
-   
-
-   
- 
-  
-   
-
-   
- 
-  
-   
-   
-   
- 
-  
-   
-   

- 
+ 
   

 
-   
+   
  

-   
-   
+
+   
  

 

  

+   
+   
+ 
+   
 

  
  


-   
- 
- 
-   
-
-   
+   
  
   

 
-   
+   
  
   

 
-   
+   
  
   

@@ -286,11 +240,6 @@
  
   

-
-   


  

svn commit: r1706294 - /uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml

[challngr]

Author: challngr
Date: Thu Oct  1 17:12:40 2015
New Revision: 1706294

URL: http://svn.apache.org/viewvc?rev=1706294=rev
Log:
UIMA-4627 Increase camel log level to warn.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml

Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml?rev=1706294=1706293=1706294=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml Thu Oct  1 
17:12:40 2015
@@ -250,11 +250,11 @@



- 
+ 

 

- 
+ 

 

svn commit: r1704907 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database: DbLoader.java DbTester.java HistoryManagerDb.java

[challngr]

Author: challngr
Date: Wed Sep 23 17:52:08 2015
New Revision: 1704907

URL: http://svn.apache.org/viewvc?rev=1704907=rev
Log:
UIMA-4577 Updates to database loader.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java?rev=1704907=1704906=1704907=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
 Wed Sep 23 17:52:08 2015
@@ -34,7 +34,6 @@ import java.util.concurrent.atomic.Atomi
 import org.apache.uima.ducc.common.Pair;
 import org.apache.uima.ducc.common.persistence.services.IStateServices;
 import org.apache.uima.ducc.common.utils.DuccLogger;
-import org.apache.uima.ducc.common.utils.Utils;
 import org.apache.uima.ducc.common.utils.id.DuccId;
 import org.apache.uima.ducc.database.DbConstants.DbCategory;
 import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
@@ -450,7 +449,8 @@ public class DbLoader
 }
 }
 
-void loadCheckpoint()
+@SuppressWarnings("unchecked")
+   void loadCheckpoint()
throws Exception
 {
 String methodName = "loadCheckpoint";
@@ -537,7 +537,7 @@ public class DbLoader
 if ( doregistry ) {
 ssd = new StateServicesDb();
 ssd.init(logger);
-if ( false ) loadServiceRegistry(serviceRegistry, false);
+loadServiceRegistry(serviceRegistry, false);
 try {
 ssd.shutdown(); 
 } catch ( Exception e ) {
@@ -547,7 +547,7 @@ public class DbLoader
 // -- Load service registry history
 ssd = new StateServicesDb();
 ssd.init(logger);
-if ( false ) loadServiceRegistry(serviceRegistryHistory, 
true);  
+loadServiceRegistry(serviceRegistryHistory, true); 
 
 }
 
 OGlobalConfiguration.USE_WAL.setValue(true);

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java?rev=1704907=1704906=1704907=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java
 Wed Sep 23 17:52:08 2015
@@ -381,7 +381,6 @@ public class DbTester
 
 // -- Load job history
 loadJobs();
-if ( true ) return;
 
 // -- Load reservation history
 loadReservations(); 

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java?rev=1704907=1704906=1704907=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
 Wed Sep 23 17:52:08 2015
@@ -525,7 +525,7 @@ public class HistoryManagerDb
 private IDuccWorkReservation restoreReservationInternal(DbHandle h, 
OrientVertex v)
 throws Exception
 {
-String methodName = "restoreReservationInternal";
+// String methodName = "restoreReservationInternal";
 IDuccWorkReservation r = null;
 
 ODocument d = v.getRecord();
@@ -537,15 +537,15 @@ public class HistoryManagerDb
 
 r  = g.fromJson(jo, DuccWorkReservation.class);
 
-List l = r.getJdReservationBeanList();
-if ( l != null ) {
-for (JdReservationBean b : l ) {
-

svn commit: r1704906 - in /uima/sandbox/uima-ducc/trunk: uima-ducc-common/src/main/java/org/apache/uima/ducc/common/ uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/ uima-ducc-web/src/main/java/or

[challngr]

Author: challngr
Date: Wed Sep 23 17:50:44 2015
New Revision: 1704906

URL: http://svn.apache.org/viewvc?rev=1704906=rev
Log:
UIMA-4569 Supply per-node quantum from NodeConfiguration.java

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java

uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java

uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java

uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java?rev=1704906=1704905=1704906=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java
 Wed Sep 23 17:50:44 2015
@@ -65,14 +65,16 @@ public class NodeConfiguration
 Map<String, DuccProperties> usermap  = new HashMap<String, 
DuccProperties>(); // all users, by name UIMA-4275
 ArrayList independentClasses = new ArrayList();
   // all classes that don't derive from something
 
-Map<String, String> allNodes   = new HashMap<String, String>();
// map node   -> nodepool name, map for dup checking
+Map<String, String> allNodes   = new HashMap<String, String>();
// map node   -> nodefile name, map for dup checking
 Map<String, DuccProperties> poolsByNodefile = new HashMap<String, 
DuccProperties>();   // nodepool node file -> nodepool props
+Map<String, DuccProperties> poolsByNodeName = new HashMap<String, 
DuccProperties>();   // Nodepools, by node
 
 Map<String, String> allImports = new HashMap<String, String>();
// map nodefile -> importer, map for dup checking
 Map<String, String> referrers  = new HashMap<String, String>();
// map nodefile -> referring nodepool, for dup checking
 
 DuccLogger logger;
 String defaultDomain = null;
+intdefaultQuantum = 15;
 String firstNodepool = null;
 
 boolean fairShareExists = false;
@@ -651,7 +653,7 @@ public class NodeConfiguration
 for (DuccProperties props : independentNodepools ) {
 String q = props.getProperty("share-quantum");
 if ( q == null ) {
-props.setProperty("share-quantum", 
""+SystemPropertyResolver.getIntProperty("ducc.rm.share.quantum", 15));
+props.setProperty("share-quantum", "" + defaultQuantum);
 } else {
 try {
Integer.parseInt(q);  // insure 
it's a number
@@ -797,6 +799,7 @@ public class NodeConfiguration
 }
 allNodes.put(node, nodefile); // for dup checking - we only 
get to read a node once
 nodes.put(node, nodefile);// UIMA-4142 map host -> domain
+poolsByNodeName.put(node, p); // So we can find pool-related 
things for the node
 
 // include fully and non-fully qualified names to allow 
sloppiness of config
 ndx = node.indexOf(".");
@@ -998,7 +1001,18 @@ public class NodeConfiguration
 return reserveDefault;
 }
 
-public int getShareQuantum(String classname)
+public int getQuantumForNode(String node)
+{
+DuccProperties np = poolsByNodeName.get(node);
+if ( np == null ) {
+// it has to be the default np, so use the default quantum
+return defaultQuantum;
+}
+// otherwise it's required that the quantum for the nodepool is set to 
something
+return Integer.parseInt(np.getProperty("share-quantum"));
+}
+
+public int getQuantumForClass(String classname)
 throws IllegalConfigurationException
 {
 // to find the quantum for a class - 
@@ -1051,6 +1065,7 @@ public class NodeConfiguration
 throw new IllegalConfigurationException("DUCC_HOME must be defined 
as a system property.");
 }
 defaultDomain = getDomainName();
+defaultQuantum = 
SystemPropertyResolver.getIntProperty("ducc.rm.share.quantum", 15);
 
 try {
 String tconfig_file_name = resolve(config_file_name);

Modified: 
uima/sandbox/uima-ducc

svn commit: r1704692 - /uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml

[challngr]

Author: challngr
Date: Tue Sep 22 19:04:31 2015
New Revision: 1704692

URL: http://svn.apache.org/viewvc?rev=1704692=rev
Log:
UIMA-4577 Update log4j to cover the db loader.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml

Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml?rev=1704692=1704691=1704692=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml Tue Sep 22 
19:04:31 2015
@@ -35,7 +35,7 @@

  

-  
+

  
  
@@ -50,16 +50,24 @@
  
   
   
+   
+ 
+ 
+   
+ 
+  
+
+  

 
 
-
-
+
+
  

  

-  
+

 
 
@@ -73,8 +81,8 @@

 
 
-
-
+
+
  

  
@@ -84,7 +92,6 @@
 
 
 
-
  

  
@@ -94,7 +101,15 @@
 
 
 
-
+ 
+   
+ 
+   
+  
+   
+
+
+
  

  
@@ -103,8 +118,8 @@

 
 
-
-
+
+
  

  
@@ -114,7 +129,6 @@
 
 
 
-
  

  
@@ -130,13 +144,24 @@
  

 
+   
+
+
+
+
+ 
+   
+ 
+   
+  
+

  
   

 

- 
+ 
   

 
@@ -146,7 +171,7 @@

 

- 
+ 
   

 
@@ -155,18 +180,28 @@
   

 
-   
+   
+ 
+  
+   
+   
+   
  
+  
+   
+   
+   
+ 
  

 
-   
+   
  
  

 

- 
+ 
  

 
@@ -185,25 +220,41 @@
   

 
-   
+   
  
-  
+  

 
-   
- 
+   
+ 
+  

 
-   
- 
+   
+ 
+  
+   
+   
+   
+ 
+  
+   
+   
+   
+ 
+  

 
-   
+   
  



- 
+ 
+   
+
+   
+ 

 

@@ -211,21 +262,35 @@
  


-   
+   
+ 
+ 
+   
+
+   
  
   

 
-   
+   
  
   

 
-   
+   
  
   

 
+   
+ 
+  
+   
+
+   


  

svn commit: r1704356 - /uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install

[challngr]

Author: challngr
Date: Mon Sep 21 16:41:39 2015
New Revision: 1704356

URL: http://svn.apache.org/viewvc?rev=1704356=rev
Log:
UIMA-4577 Allow --db-password for post install so it can be run automated from 
a script.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install?rev=1704356=1704355=1704356=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install Mon Sep 21 
16:41:39 2015
@@ -41,7 +41,7 @@ import database as db
 
 class PostInstall():
 
-def usage(self, msg):
+def usage(self, *msg):
 
 if ( msg != None ):
 print ' '.join(msg)
@@ -61,6 +61,9 @@ class PostInstall():
 print "   [-j, --jvm] "
 print "This is the full path to java command to be used to 
start DUCC; e.g., /usr/bin/java"
 print ""
+print "   [-d --db-password] "
+print "This is the password DUCC uses to manage the database."
+print ""
 print "   [-h, -? --help]"
 print "Prints this message."
 print ""
@@ -90,12 +93,13 @@ class PostInstall():
 print 'Configuring database at', dburl, 'from', dbconfig 
 print ''
 
-dbpw = ''
-while ( dbpw == '' ):
-dbpw = raw_input('Enter password for database root:')
+if ( self.database_pw == None ):
+self.database_pw = ''
+while ( self.database_pw == '' ):
+self.database_pw = raw_input('Enter password for database 
root:')
 
 print ''
-db.configure(dbconfig, self.ducc_head, dbpw)
+db.configure(dbconfig, self.ducc_head, self.database_pw)
 
 
 def get_keystore_pw(self, prompt):
@@ -329,10 +333,11 @@ class PostInstall():
 
 self.ducc_head = None
 self.keystore_pw = None
+self.database_pw = None
 self.path_to_java = None
 
 try:
-opts, args = getopt.getopt(argv, 'j:k:n:h?', ['jvm=', 'keystore=', 
'head-node=', 'help'])
+opts, args = getopt.getopt(argv, 'd:j:k:n:h?', ['db-password=', 
'jvm=', 'keystore=', 'head-node=', 'help'])
 except:
 self.usage("Invalid arguments " + ' '.join(argv))
 
@@ -340,12 +345,14 @@ class PostInstall():
 for ( o, a ) in opts:
 if o in ('-n', '--head-node'):
 self.ducc_head = a
+if o in ('-d', '--db-password'):
+self.database_pw = a
 if o in ('-k', '--keystore'):
 self.keystore_pw = a
 if o in ('-j', '--jvm'):
 self.path_to_java = a
 elif o in ('-h', '-?', '--help'):
-usage(None)
+self.usage(None)
 
 self.DUCC_HOME = find_ducc_home()
 self.localhost = find_localhost()

svn commit: r1703913 - /uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java

[challngr]

Author: challngr
Date: Fri Sep 18 19:18:51 2015
New Revision: 1703913

URL: http://svn.apache.org/viewvc?rev=1703913=rev
Log:
UIMA-4569 Fix wording when something can't be recovered from history.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java?rev=1703913=1703912=1703913=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java
 Fri Sep 18 19:18:51 2015
@@ -119,7 +119,7 @@ public class DuccData {
 map.put(duccId, history);
 }
 } catch ( Exception e ) {
-logger.warn(methodName, duccId, "Cannot recover", 
duccWork.getDuccType(), "from database");
+logger.warn(methodName, duccId, "Cannot recover", 
duccWork.getDuccType(), "from history.");
 }
}
}

svn commit: r1703877 - /uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java

[challngr]

Author: challngr
Date: Fri Sep 18 14:54:52 2015
New Revision: 1703877

URL: http://svn.apache.org/viewvc?rev=1703877=rev
Log:
UIMA-4577 Set logger so things get logged in higher level components correctly.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java?rev=1703877=1703876=1703877=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
 Fri Sep 18 14:54:52 2015
@@ -7,6 +7,7 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 
+import org.apache.uima.ducc.common.main.DuccService;
 import org.apache.uima.ducc.common.utils.DuccLogger;
 import org.apache.uima.ducc.common.utils.DuccProperties;
 import org.apache.uima.ducc.database.DbConstants.DbCategory;
@@ -24,7 +25,8 @@ import com.tinkerpop.blueprints.impls.or
 
 public class DbHandle
 {
-private DuccLogger logger = DuccLogger.getLogger(DbHandle.class, "DB");  
// get the component logger
+//private DuccLogger logger = DuccLogger.getLogger(DbHandle.class, "DB");  
// get the component logger
+private DuccLogger logger = 
DuccService.getDuccLogger(DbHandle.class.getName());
 
 DbManager manager;
 public OrientBaseGraph graphDb;

svn commit: r1703896 - in /uima/sandbox/uima-ducc/trunk: src/main/admin/ uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/

[challngr]

Author: challngr
Date: Fri Sep 18 16:05:20 2015
New Revision: 1703896

URL: http://svn.apache.org/viewvc?rev=1703896=rev
Log:
UIMA-4569 Adjust viz for memory-only reported by OR.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py

uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/Markup.java

uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java

uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1703896=1703895=1703896=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Fri Sep 18 
16:05:20 2015
@@ -215,9 +215,9 @@ class DuccUtil(DuccBase):
 return False
 
 resp = conn.getresponse()
-#print 'response code', resp.status, resp.reason
+print 'response code', resp.status, resp.reason
 data = resp.read()
-#print 'Data:', data
+print 'Data:', data
 
 if ( resp.status == 200 ):
 # it will be simple json that Python will see as lists and maps so 
we can just eval it
@@ -293,7 +293,7 @@ class DuccUtil(DuccBase):
 
 classpath = self.DUCC_HOME + '/lib/uima-ducc/*:' + classpath
 dburl = self.ducc_properties.get('ducc.state.database.url') 
-cmd = ' '.join([self.java(), 
'-DDUCC_HOME=/home/challngr/ducc_runtime_db', '-cp', classpath, main, dburl])
+cmd = ' '.join([self.java(), '-DDUCC_HOME=' + self.DUCC_HOME, '-cp', 
classpath, main, dburl])
 print cmd
 self.spawn(cmd)
 

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/Markup.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/Markup.java?rev=1703896=1703895=1703896=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/Markup.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/Markup.java
 Fri Sep 18 16:05:20 2015
@@ -215,8 +215,7 @@ public class Markup
 break;
 case Undefined:
 out.append("");
-out.append(j.qshares); 
-out.append(" unused shares (");
+out.append("mem avail (");
 out.append((j.qshares * j.quantum));
 out.append("GB) on ");
 out.append(h.name); 

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java?rev=1703896=1703895=1703896=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java
 Fri Sep 18 16:05:20 2015
@@ -146,15 +146,8 @@ public class NodeViz
 NodeConfiguration nc = new NodeConfiguration(class_definitions, null, 
user_registry, logger);// UIMA-4142 make the config global
 try {
nc.readConfiguration();
-   } catch (FileNotFoundException e1) {
-   // TODO Auto-generated catch block
-   e1.printStackTrace();
-   } catch (IOException e1) {
-   // TODO Auto-generated catch block
-   e1.printStackTrace();
-   } catch (IllegalConfigurationException e1) {
-   // TODO Auto-generated catch block
-   e1.printStackTrace();
+   } catch (Exception e ) {
+logger.error(methodName, null, "Cannot read node configuration.  
Some information may not be quite right.");
}
 
 // first step, generate the viz from the OR map which seems to have 
everything we need
@@ -202,7 +195,7 @@ public class NodeViz
 // but punt and try not to crash.
 logger.warn(methodName, null, "Cannot find scheduling class or 
quantum for " + sclass + ". Using default quantum of " + default_quantum);
 }
-  

svn commit: r1703684 - in /uima/sandbox/uima-ducc/trunk: src/main/admin/ uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/ uima-ducc-database/src/main/java/org/apache/ui

[challngr]

Author: challngr
Date: Thu Sep 17 20:34:08 2015
New Revision: 1703684

URL: http://svn.apache.org/viewvc?rev=1703684=rev
Log:
UIMA-4577 Scripting fixes.  Fix leak in loader and speed it up.  Add indexes.  
Minor tupos. Go to ODB 2.1.2
  to pick up indexing fix.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/db_console

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbConstants.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
uima/sandbox/uima-ducc/trunk/uima-ducc-parent/pom.xml

uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/db_console
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_console?rev=1703684=1703683=1703684=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db_console (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_console Thu Sep 17 20:34:08 
2015
@@ -38,8 +38,11 @@ class DbConsole(DuccUtil):
 print "Note that Python must be at least version 2.6 but not 3.x.  
You are running version", sys.version_info
 return
 
+if ( self.db_parms == '--disabled--' ):
+print "Database is disabled."
+return
 
-(jvm_parms, classpath, db_rt, dburl, dbrest, dbroot) = self.db_parms()
+(jvm_parms, classpath, db_rt, dburl, dbrest, dbroot) = self.db_parms
 
 
 main = 'com.orientechnologies.orient.graph.console.OGremlinConsole'

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java?rev=1703684=1703683=1703684=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java
 Thu Sep 17 20:34:08 2015
@@ -31,6 +31,10 @@ public interface IStateServices {
public static String svc_reg_dir = IDuccEnv.DUCC_STATE_SVCREG_DIR;
public static String svc_hist_dir = IDuccEnv.DUCC_HISTORY_SVCREG_DIR;
 
+
+public static final String archive_key = "is_archived";
+public static final String archive_flag = "true";
+
public static final String svc = "svc";
public static final String meta = "meta";
 

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbConstants.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbConstants.java?rev=1703684=1703683=1703684=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbConstants.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbConstants.java
 Thu Sep 17 20:34:08 2015
@@ -38,6 +38,16 @@ public interface DbConstants
 ;
 }
 
+// Every vertex must inherit from here so we can use common indexes
+public enum DuccVertexBase
+implements Schema
+{
+VBase {
+public String pname() { return "VDuccBase"; } 
+},
+;
+}
+
 public enum DbVertex
 implements Schema
 {
@@ -88,15 +98,25 @@ public interface DbConstants
 
 }
 
+public enum DuccEdgeBase
+implements Schema
+{
+EdgeBase {
+public String pname() { return "ducc_ebase"; } 
+},
+;
+}
+
 public enum DbEdge
 implements Schema
 {
 /

svn commit: r1703203 [3/5] - in /uima/sandbox/uima-ducc/trunk: ./ src/main/admin/ src/main/assembly/ src/main/resources/ src/main/resources/private/ uima-ducc-common/src/main/java/org/apache/uima/ducc

[challngr]

Added: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java?rev=1703203=auto
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java
 (added)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java
 Tue Sep 15 14:31:24 2015
@@ -0,0 +1,592 @@
+package org.apache.uima.ducc.database;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.uima.ducc.common.persistence.services.IStateServices;
+import org.apache.uima.ducc.common.utils.DuccLogger;
+import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkService;
+
+/**
+ * Toy orientdb loader to load a historydb from ducc history
+ */
+
+public class DbTester
+{
+DuccLogger logger = DuccLogger.getLogger(DbTester.class, "DBLOAD");
+
+HistoryManagerDb hmd = null;
+StateServicesDb  ssd = null;
+
+String history_url = "remote:localhost/DuccHistory";
+String state_url   = "remote:localhost/DuccState";
+
+// String jobHistory = System.getProperty("user.home") + 
"/ducc_runtime/history/jobs";
+String jobHistory = "/home/ducc/ducc_runtime/history/jobs";
+
+// String reservationHistory = System.getProperty("user.home") + 
"/ducc_runtime/history/reservations";
+String reservationHistory = "/home/ducc/ducc_runtime/history/reservations";
+
+//String serviceHistory = System.getProperty("user.home") + 
"/ducc_runtime/history/services";
+String serviceHistory = "/home/ducc/ducc_runtime/history/services";
+
+//String serviceHistory = System.getProperty("user.home") + 
"/ducc_runtime/history/services";
+String serviceRegistryHistory = 
"/home/ducc/ducc_runtime/history/services-registry";
+
+//String serviceRegistry = System.getProperty("user.home") + 
"/ducc_runtime/state/services";
+String serviceRegistry = "/home/ducc/ducc_runtime/state/services";
+
+int nthreads = 20;
+AtomicInteger counter = new AtomicInteger(0);
+
+public DbTester()
+throws Exception
+{
+System.setProperty("ducc.history.database.url", history_url);
+System.setProperty("ducc.state.database.url", state_url);
+}
+
+void closeStream(InputStream in)
+{
+try { in.close(); } catch(Exception e) {}
+}
+
+public void loadJobs()
+{
+String methodName = "loadJobs";
+LinkedBlockingQueue jobqueue = new 
LinkedBlockingQueue();
+
+int max_to_load = 2;
+int nth = Math.min(nthreads, max_to_load);
+JobLoader[] loader = new JobLoader[nth];
+Thread[]threads = new Thread[nth];
+List ids = new ArrayList();
+
+for ( int i = 0; i < nth; i++ ) {
+loader[i] = new JobLoader(jobqueue, ids);
+threads[i] = new Thread(loader[i]);
+threads[i].start();
+}
+
+File dir = new File(jobHistory);
+File[] files = dir.listFiles();
+logger.info(methodName, null, "Reading", files.length, "jobs.");
+
+int c = 0;
+for ( File f : files) {
+String s = f.toString();
+if ( s.endsWith(".dwj") ) {
+logger.info(methodName, null, "Loading file", c++, ":", f);
+IDuccWorkJob job = null;
+FileInputStream fis = null;
+ObjectInputStream in = null;
+
+try {
+long now = System.currentTimeMillis();
+fis = new FileInputStream(f);
+in = new ObjectInputStream(fis);
+job =  (IDuccWorkJob) in.readObject();
+logger.info(methodName, job.getDuccId(), "Time to read 
job:", System.currentTimeMillis() - now);
+jobqueue.offer(job);
+counter.getAndIncrement();
+} catch(Exception e) {
+logger.info(methodName, null, e);
+} finally {
+// restoreJob(job.getDuccId().getFriendly());
+closeStream(in);
+closeStream(fis);
+if ( c >= max_to_load ) break;
+}
+} else {
+logger.info(methodName, null, "Can't find 

svn commit: r1703203 [2/5] - in /uima/sandbox/uima-ducc/trunk: ./ src/main/admin/ src/main/assembly/ src/main/resources/ src/main/resources/private/ uima-ducc-common/src/main/java/org/apache/uima/ducc

[challngr]

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesFactory.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesFactory.java?rev=1703203=1703202=1703203=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesFactory.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesFactory.java
 Tue Sep 15 14:31:24 2015
@@ -18,14 +18,61 @@
 */
 package org.apache.uima.ducc.common.persistence.services;
 
+import org.apache.uima.ducc.common.main.DuccService;
+import org.apache.uima.ducc.common.utils.DuccLogger;
 
 
-public class StateServicesFactory {
 
-   private static IStateServices instance = new StateServices();
-   
-   public static IStateServices getInstance() {
-   return instance;
+public class StateServicesFactory 
+{
+private static IStateServices instance = null;
+
+   private static IStateServices getInstanceInternal(String callerClass, 
String component) 
+{
+String methodName = "getInstance";
+   // log4j logging annoyance.  We require the caller to give us 
its base package so
+// we can configure a logger that writes to the right appender
+   // log4j logging annoyance.  We require the caller to give us its base 
package so
+// we can configure a logger that writes to the right appender
+int ndx = callerClass.lastIndexOf(".");
+String stem = callerClass.substring(0, ndx);
+
+String clname = System.getProperty("ducc.service.persistence.impl");
+if ( clname == null ) {
+   DuccLogger logger = DuccService.getDuccLogger();
+   logger.warn(methodName, null, "Service persistence manager is 
not configured.  Returning null instance.");
+return new NullStateServices();
+}
+ndx = clname.lastIndexOf(".");
+String clfile = clname.substring(ndx+1);
+//
+// We try to construct the persistence object.  If it fails, we return 
a
+// "null" object conforming to the interface but doing nothing to 
hopefully
+// reduce NPEs.
+//
+DuccLogger logger = DuccLogger.getLogger(stem + "." + clfile, "DB");  
// get the component logger
+
+IStateServices ret = null;
+try {
+@SuppressWarnings("unchecked")
+   Class iss = 
(Class) Class.forName(clname);
+ret = (IStateServices) iss.newInstance();
+ret.init(logger);
+} catch ( Throwable t ) {
+logger.error(methodName, null, "Cannot instantiate service 
persistence class", clname, ":", t);
+ret = new NullStateServices();
+}
+
+return ret;
+   }
+
+   public static IStateServices getInstance(String callerClass, String 
component) 
+{
+if ( instance == null ) {
+instance = getInstanceInternal(callerClass, component);
+}
+
+return instance;
}

 }

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesSet.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesSet.java?rev=1703203=1703202=1703203=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesSet.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesSet.java
 Tue Sep 15 14:31:24 2015
@@ -19,17 +19,18 @@
 package org.apache.uima.ducc.common.persistence.services;
 
 import java.util.HashMap;
-import java.util.Properties;
+
+import org.apache.uima.ducc.common.utils.DuccProperties;
 
 public class StateServicesSet {
 
-   private HashMap map = new 
HashMap();
+   private HashMap map = new HashMap();

-   public void put(String key, Properties value) {
+   public void put(String key, DuccProperties value) {
map.put(key, value);
}

-   public Properties get(String key) {
+   public DuccProperties get(String key) {
return map.get(key);
}
 }

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccProperties.java
URL: 

svn commit: r1703203 [1/5] - in /uima/sandbox/uima-ducc/trunk: ./ src/main/admin/ src/main/assembly/ src/main/resources/ src/main/resources/private/ uima-ducc-common/src/main/java/org/apache/uima/ducc

[challngr]

Author: challngr
Date: Tue Sep 15 14:31:24 2015
New Revision: 1703203

URL: http://svn.apache.org/r1703203
Log:
UIMA-4577 First delivery, DB project.

Added:
uima/sandbox/uima-ducc/trunk/src/main/admin/database.py   (with props)
uima/sandbox/uima-ducc/trunk/src/main/admin/db.py   (with props)
uima/sandbox/uima-ducc/trunk/src/main/admin/db_console   (with props)
uima/sandbox/uima-ducc/trunk/src/main/resources/private/database.xml

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/pom.xml
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbConstants.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbObject.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ProcessToJobList.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/DbQuery.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/Server.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/Shutdown.java

uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/SmLoader.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/resources/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/test/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/test/java/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/test/resources/

uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java
Modified:
uima/sandbox/uima-ducc/trunk/pom.xml
uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc
uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml
uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccEnv.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesFactory.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesSet.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccProperties.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/java/org/apache/uima/ducc/common/test/cmd

svn commit: r1703203 [5/5] - in /uima/sandbox/uima-ducc/trunk: ./ src/main/admin/ src/main/assembly/ src/main/resources/ src/main/resources/private/ uima-ducc-common/src/main/java/org/apache/uima/ducc

[challngr]

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccPerWorkItemStatistics.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccPerWorkItemStatistics.java?rev=1703203=1703202=1703203=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccPerWorkItemStatistics.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccPerWorkItemStatistics.java
 Tue Sep 15 14:31:24 2015
@@ -26,4 +26,5 @@ public interface IDuccPerWorkItemStatist
public double getMin();
public double getMean();
public double getStandardDeviation();
+   public void   setStandardDeviation(double s);
 }

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java?rev=1703203=1703202=1703203=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java
 Tue Sep 15 14:31:24 2015
@@ -18,15 +18,62 @@
 */
 package org.apache.uima.ducc.transport.event.common.history;
 
+import org.apache.uima.ducc.common.main.DuccService;
+import org.apache.uima.ducc.common.persistence.services.IStateServices;
+import org.apache.uima.ducc.common.utils.DuccLogger;
+
 
 
 public class HistoryFactory 
 {
 
-   private static IHistoryPersistenceManager instance = new 
HistoryPersistenceManager();
+   private static IHistoryPersistenceManager instance = null; //new 
HistoryPersistenceManager();

-   public static IHistoryPersistenceManager getInstance() {
-   return instance;
+   public static IHistoryPersistenceManager getInstance(String 
callerClass) 
+{
+if ( instance != null ) return instance;
+
+String methodName = "getInstance";
+
+// log4j logging annoyance.  We require the caller to give us its base 
package so
+// we can configure a logger that writes to the right appender
+int ndx = callerClass.lastIndexOf(".");
+String stem = callerClass.substring(0, ndx);
+
+String clname = System.getProperty("ducc.job.history.impl");
+
+if ( clname == null ) {
+DuccLogger logger = DuccService.getDuccLogger();
+logger.error(methodName, null, "Job history class is not 
configured.");
+instance = new NullHistoryManager();
+return instance;
+} 
+
+ndx = clname.lastIndexOf(".");
+String clfile = clname.substring(ndx+1);
+//
+// We try to construct the persistence object.  If it fails, we return 
a
+// "null" object conforming to the interface but doing nothing to 
hopefully
+// reduce NPEs.
+//
+DuccLogger logger = DuccLogger.getLogger(stem + "." + clfile, "DB");  
// get the component logger
+
+//
+// We try to construct the persistence object.  If it fails, we return 
a
+// "null" object conforming to the interface but doing nothing to 
hopefully
+// reduce NPEs.
+//
+
+try {
+@SuppressWarnings("unchecked")
+   Class iss = 
(Class) Class.forName(clname);
+instance = (IHistoryPersistenceManager) iss.newInstance();
+instance.setLogger(logger);
+} catch ( Throwable t ) {
+logger.error(methodName, null, "Cannot instantiate service 
persistence class", clname, ":", t);
+instance = new NullHistoryManager();
+}
+return instance;
}

 }

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java?rev=1703203=1703202=1703203=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java
 (original)
+++ 

svn commit: r1703203 [4/5] - in /uima/sandbox/uima-ducc/trunk: ./ src/main/admin/ src/main/assembly/ src/main/resources/ src/main/resources/private/ uima-ducc-common/src/main/java/org/apache/uima/ducc

[challngr]

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/runducc
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/runducc?rev=1703203=1703202=1703203=diff
==
--- uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/runducc 
(original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/runducc 
Tue Sep 15 14:31:24 2015
@@ -70,7 +70,10 @@ class DuccProcess(Thread):
 for line in f:
 toks = line.strip().split('=');
 if ( toks[0].strip() == 'threads' ):
-threads = toks[1]
+if ( self.runner.thread_override == None ):
+threads = toks[1]
+else:
+threads = self.runner.thread_override
 elif (toks[0].strip() == 'class'):
 clz = toks[1]
 elif (toks[0].strip() == 'memory'):
@@ -555,12 +558,15 @@ class RunDucc(DuccUtil):
 print '   --init_fail_cap number-of-failures.'
 print '   This is the max init failures tolerated before the 
system starts to cap processes. Default:', self.init_fail_cap
 print ''
-print '   -IB'
+print '   --IB'
 print '   The JP will leak in init() until DUCC (hopefully) kills 
us'
 print ''
-print '   -PB'
+print '   --PB'
 print '   The JP will leak in process() until DUCC (hopefully) 
kills us'
 print ''
+print '   --TO number-of-threads'
+print '   Thread-override: force this number of threads regardless 
of what is in job spec.'
+print ''
 print '   -r, --range seconds'
 print '   This is the AE initializion time range over base in 
milliseconds. Default:', self.init_range
 print '   Init time is -i value + random[0, -rvalue]'
@@ -643,13 +649,14 @@ class RunDucc(DuccUtil):
 self.max_machines = 0
 self.use_http = False
 self.descriptor_as_file = False
+self.thread_override = None
 
 try:
 opts, args  = getopt.getopt(argv, 
'b:d:fi:m:n:op:q:r:s:t:u:v:w:x:y:z:?h', ['AE', 'DD', 'file', 'SE=', 'IB=', 
'PB=', 'directory=', 'batchfile=', 'init_time=',
 
'init_fail_cap=', 'range=', 'memory_override=', 'nmachines=', 
'process_timeout=', 
 
'init_timeout=', 'observe'
 
'jd_uima_log=', 'jp_uima_log=',
-'http',
+
'http', 'threads='
   ])
 except:
 print "Unknown option"
@@ -698,6 +705,8 @@ class RunDucc(DuccUtil):
 self.init_bloat = a
 elif o in ('--PB'):
 self.process_bloat = a
+elif o in ('--threads'):
+self.thread_override = a
 elif ( o == '-q'):
 self.cr_getnext_bloat = int(a)
 elif ( o == '-s'):
@@ -752,6 +761,7 @@ class RunDucc(DuccUtil):
 print 'jd_uima_log:', self.jd_uima_log
 print 'jp_uima_log:', self.jp_uima_log
 print 'DUCC_HOME  :', self.DUCC_HOME
+print 'Thread override:', self.thread_override
 
 self.submit_package = 'org.apache.uima.ducc'
 

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim?rev=1703203=1703202=1703203=diff
==
--- uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim 
(original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim 
Tue Sep 15 14:31:24 2015
@@ -44,6 +44,10 @@ from properties import Properties
 from ducc   import Ducc
 from ducc_util  import ThreadPool
 
+# multi-threaded start can overwhelm ssh if going to the same target host.  We 
inject
+# a short sleep between starts to make it better.  This is how long to sleep.
+global SLEEP_TIME
+SLEEP_TIME = .1
 
 class StartSim(DuccUtil):
 
@@ -58,17 +62,6 @@ class StartSim(DuccUtil):
 broker_host = self.ducc_properties.get('ducc.broker.hostname')
 print 'broker host', broker_host
 lines = self.ssh(broker_host, True, "'", self.DUCC_HOME + 
'/admin/ducc.py', '-c', 'broker', "'")
-while 1:
-line = lines.readline().strip()
-if ( not line ):
-break
-#print '[] ' + line
-if ( 

svn commit: r1697733 - in /uima/sandbox/uima-ducc/trunk: uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/

[challngr]

Author: challngr
Date: Tue Aug 25 16:49:56 2015
New Revision: 1697733

URL: http://svn.apache.org/r1697733
Log:
UIMA-4569 RM should report allocations in GB instead of shares - eliminate 
warnings in eclipse.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java?rev=1697733r1=1697732r2=1697733view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java
 Tue Aug 25 16:49:56 2015
@@ -654,7 +654,7 @@ public class NodeConfiguration
 props.setProperty(share-quantum, 
+SystemPropertyResolver.getIntProperty(ducc.rm.share.quantum, 15));
 } else {
 try {
-int quantum = Integer.parseInt(q);
+   Integer.parseInt(q);  // insure 
it's a number
 } catch (NumberFormatException e) {
 throw new IllegalConfigurationException(Value for 
\share-quantum\ in nodepool  + props.getProperty(name) +  is not a 
number.);
 }

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java?rev=1697733r1=1697732r2=1697733view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
 Tue Aug 25 16:49:56 2015
@@ -741,11 +741,6 @@ class NodePool
String methodName = rearrangeVirtual;
 if ( allMachines.containsKey(m.key()) ) {
 int v_order = m.getVirtualShareOrder();
-
-if ( v_order  0 ) {
-   int stop_here = 1;
-   stop_here++;
-}
 int r_order = m.getShareOrder();
 
 logger.trace(methodName, null, m.getId(), order, order, 
v_order, v_order, r_order, r_order);

svn commit: r1697485 - in /uima/sandbox/uima-ducc/trunk: uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ uima-ducc-transport/src/main/

[challngr]

Author: challngr
Date: Mon Aug 24 19:25:05 2015
New Revision: 1697485

URL: http://svn.apache.org/r1697485
Log:
UIMA-4569 RM should report allocations in GB instead of shares.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IRmJob.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ISchedulerMain.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/User.java

uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/RmStateDuccEvent.java

uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/rm/IRmJobState.java

uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/rm/Resource.java

uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/rm/RmJobState.java

uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/JobFragment.java

uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/Markup.java

uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java

uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java?rev=1697485r1=1697484r2=1697485view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
 Mon Aug 24 19:25:05 2015
@@ -124,7 +124,6 @@ public class JobManagerConverter
 {
String methodName = blacklistJob;
 
-int order = scheduler.calcShareOrder(memory);
 MapDuccId, IResource all_shares  = null; 
 MapDuccId, IResource shrunken_shares = null;
 MapDuccId, IResource expanded_shares = null;
@@ -137,16 +136,18 @@ public class JobManagerConverter
 
 // first time - everything must go
 IDuccProcessMap pm = ((IDuccWorkExecutable)job).getProcessMap();   
   
+int quantum = 0;
 for ( IDuccProcess proc : pm.values() ) {  // build up Shares 
from the incoming state
 NodeIdentity ni = proc.getNodeIdentity();
 Machine m = scheduler.getMachine(ni);
 int share_order = 1;
+quantum = m.getQuantum();
 
 if ( m != null ) {
 if ( proc.isActive() || (proc.getProcessState() == 
ProcessState.Undefined) ) {
 logger.info(methodName, job.getDuccId(), blacklist, 
proc.getDuccId(), 
 state, proc.getProcessState(), isActive, 
proc.isActive(), isComplete, proc.isComplete());
-m.blacklist(job.getDuccId(), proc.getDuccId(), order);
+m.blacklist(job.getDuccId(), proc.getDuccId(), memory);
 if ( evict ) {
 share_order = m.getShareOrder(); // best guess
 Resource r = new Resource(proc.getDuccId(), 
proc.getNode(), false, share_order, 0);
@@ -162,7 +163,7 @@ public class JobManagerConverter
 }
 
 if ( evict  (shrunken_shares.size()  0) ) {
-RmJobState rjs = new RmJobState(job.getDuccId(), all_shares, 
shrunken_shares, expanded_shares);
+RmJobState rjs = new RmJobState(job.getDuccId(), quantum, 
all_shares, shrunken_shares, expanded_shares);
 rjs.setDuccType(job.getDuccType

svn commit: r1697497 - /uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java

[challngr]

Author: challngr
Date: Mon Aug 24 20:23:32 2015
New Revision: 1697497

URL: http://svn.apache.org/r1697497
Log:
UIMA-4569 RM should report allocations in GB instead of shares.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java?rev=1697497r1=1697496r2=1697497view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java
 Mon Aug 24 20:23:32 2015
@@ -144,6 +144,7 @@ public class NodeConfiguration
 defaultNodepool.put(name, optional);
 defaultNodepool.put(nodefile, optional);
 defaultNodepool.put(parent, optional);
+defaultNodepool.put(share-quantum, optional);
 defaultNodepool.put(domain, optional);
 defaultNodepool.put(search-order, 100);// temporary.  UIMA-4324
 
@@ -641,6 +642,37 @@ public class NodeConfiguration
 }
 
 /**
+ * Look at all top-level nodepools and insure the scheduling quantum is 
set.  If not, inherit it from
+ * ducc.properties.
+ */
+void setShareQuantum()
+   throws IllegalConfigurationException
+{
+for (DuccProperties props : independentNodepools ) {
+String q = props.getProperty(share-quantum);
+if ( q == null ) {
+props.setProperty(share-quantum, 
+SystemPropertyResolver.getIntProperty(ducc.rm.share.quantum, 15));
+} else {
+try {
+int quantum = Integer.parseInt(q);
+} catch (NumberFormatException e) {
+throw new IllegalConfigurationException(Value for 
\share-quantum\ in nodepool  + props.getProperty(name) +  is not a 
number.);
+}
+}
+}
+}
+
+/**
+ * (Recursively) walk up parental chain to find the top-level np for this 
guy.
+ */
+DuccProperties getTopLevel(DuccProperties child)
+{
+String parent = child.getStringProperty(parent, null);
+if ( parent == null ) return child;
+return getTopLevel(nodepools.get(parent));
+}
+
+/**
  * Find all the top-level nodepools.
  * Make sure every parent nodepool exists.
  * Set the names of the child nodepools into each parent.
@@ -650,12 +682,26 @@ public class NodeConfiguration
 throws IllegalConfigurationException
 {
 
-// map the child nodepools into their parents
+// Insure default is set right
+setShareQuantum();
+
+// Map the child nodepools into their parents
 for ( DuccProperties p : nodepools.values() ) {
 String parent = p.getStringProperty(parent, null);
 String name   = p.getStringProperty(name);
 
 if ( parent != null ) {
+
+// Insure no scheduling quantum is set
+if ( p.getProperty(share-quantum) != null ) {
+throw new IllegalConfigurationException(Nodepool  + name 
+ : \share-quantum\ is legal only for top-level nodepools.);
+}
+
+// Now pull down the parent's scheduling quantum
+DuccProperties tl = getTopLevel(p);
+p.setProperty(share-quantum, 
tl.getProperty(share-quantum));  // (guaranteed non-null by 
setSchedulingQuantum)
+
+// And now connect children and parents
 DuccProperties par_pool = nodepools.get(parent);
 if ( par_pool == null ) {
 throw new IllegalConfigurationException(Nodepool  + 
name+  parent pool  + parent +  cannot be found.);
@@ -952,6 +998,23 @@ public class NodeConfiguration
 return reserveDefault;
 }
 
+public int getShareQuantum(String classname)
+throws IllegalConfigurationException
+{
+// to find the quantum for a class - 
+// -- look up the class to get the nodepool name
+// -- look in the nodepool for the quantum
+// must throw on invalid classname which could be a tupo on the part 
of whoever calls this
+DuccProperties props = clmap.get(classname);
+if ( props == null ) {
+throw new IllegalConfigurationException(Class  + classname +  
is not configured.);
+}
+
+String npname = props.getProperty(nodepool);
+DuccProperties np = nodepools.get(npname);
+return Integer.parseInt(np.getProperty(share-quantum));
+}
+
 public DuccProperties[] getToplevelNodepools

svn commit: r1693914 - /uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java

[challngr]

Author: challngr
Date: Mon Aug  3 15:17:49 2015
New Revision: 1693914

URL: http://svn.apache.org/r1693914
Log:
UIMA-4358 Use orchestrator publish rate, not the deprecated abbreviate 
orchestrator rate to trigger scheduling.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java?rev=1693914r1=1693913r2=1693914view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java
 Mon Aug  3 15:17:49 2015
@@ -209,7 +209,7 @@ public class ResourceManagerComponent
 nodeStability = 
SystemPropertyResolver.getIntProperty(ducc.rm.node.stability, 
DEFAULT_STABILITY_COUNT);
 nodeMetricsUpdateRate = 
SystemPropertyResolver.getIntProperty(ducc.agent.node.metrics.publish.rate, 
DEFAULT_NODE_METRICS_RATE);
 schedulingRatio   = 
SystemPropertyResolver.getIntProperty(ducc.rm.state.publish.ratio, 
DEFAULT_SCHEDULING_RATIO);
-orPublishingRate  = 
SystemPropertyResolver.getIntProperty(ducc.orchestrator.abbreviated.state.publish.rate,
 DEFAULT_OR_PUBLISH_RATE);
+orPublishingRate  = 
SystemPropertyResolver.getIntProperty(ducc.orchestrator.state.publish.rate, 
DEFAULT_OR_PUBLISH_RATE);
 minRmPublishingRate   = orPublishingRate - DEFAULT_RM_PUBLISHING_SLOP;
 if ( minRmPublishingRate =0 ) minRmPublishingRate = 
DEFAULT_RM_PUBLISHING_SLOP;// somewhat arbitrary, but what else?
 

svn commit: r1693966 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler: ResourceClass.java RmJob.java

[challngr]

Author: challngr
Date: Mon Aug  3 20:15:18 2015
New Revision: 1693966

URL: http://svn.apache.org/r1693966
Log:
UIMA-4358 Fix bad math calculating allotments.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java?rev=1693966r1=1693965r2=1693966view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java
 Mon Aug  3 20:15:18 2015
@@ -111,11 +111,10 @@ public class ResourceClass
 this.enforce_memory = props.getBooleanProperty(enforce, true);
 }
 
+// For now, R 2.0.0 not configurable, and not cappable.  Hope to 
revive in future release.
 this.fair_share_cap = Integer.MAX_VALUE;  // UIMA-4275
 
 if ( this.policy == Policy.FAIR_SHARE ) {
-fair_share_cap = props.getIntProperty(cap, Integer.MAX_VALUE);
-if (fair_share_cap == 0) fair_share_cap = Integer.MAX_VALUE;
 
 this.share_weight = props.getIntProperty(weight);
 if ( props.containsKey(expand-by-doubling) ) {
@@ -337,10 +336,11 @@ public class ResourceClass
 // UIMA-4275
 public boolean fairShareCapExceeded(IRmJob j)
 {
-if ( policy != Policy.FAIR_SHARE ) return false;
-
-if ( j.getShareOrder() + countActiveShares()  calculateCap() ) return 
true;
 return false;
+// if ( policy != Policy.FAIR_SHARE ) return false;
+
+// if ( j.getShareOrder() + countActiveShares()  calculateCap() ) 
return true;
+// return false;
 }
 
 /**

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java?rev=1693966r1=1693965r2=1693966view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
 Mon Aug  3 20:15:18 2015
@@ -342,7 +342,7 @@ public class RmJob
 public int countOccupancy()
 {
 
-return assignedShares.size() + pendingShares.size();
+return (assignedShares.size() + pendingShares.size()) * share_order;
 
 // if ( (given_by_order == null) || (given_by_order[share_order] == 0) 
) {
 // // must use current allocation because we haven't been counted 
yet

svn commit: r1691596 - /uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/randomsleep/FixedSleepAE.java

[challngr]

Author: challngr
Date: Fri Jul 17 16:00:48 2015
New Revision: 1691596

URL: http://svn.apache.org/r1691596
Log:
UIMA-4358 Delete deprecated code.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/randomsleep/FixedSleepAE.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/randomsleep/FixedSleepAE.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/randomsleep/FixedSleepAE.java?rev=1691596r1=1691595r2=1691596view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/randomsleep/FixedSleepAE.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/randomsleep/FixedSleepAE.java
 Fri Jul 17 16:00:48 2015
@@ -346,7 +346,7 @@ public class FixedSleepAE extends CasAnn
 int   total  = Integer.parseInt(tok.nextToken());
 doubleerror_rate = getDoubleFromEnv(AE_RUNTIME_ERROR, false);
 doubleexit_rate  = getDoubleFromEnv(AE_RUNTIME_EXIT, false);
-Stringlogid  = tok.nextToken();
+//Stringlogid  = tok.nextToken();
 
 RuntimeMXBean rmxb   = ManagementFactory.getRuntimeMXBean();
 Stringpid= rmxb.getName();

svn commit: r1691594 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler: NodePool.java NodepoolScheduler.java

[challngr]

Author: challngr
Date: Fri Jul 17 15:58:13 2015
New Revision: 1691594

URL: http://svn.apache.org/r1691594
Log:
UIMA-4358 Delete deprecated code.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java?rev=1691594r1=1691593r2=1691594view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
 Fri Jul 17 15:58:13 2015
@@ -19,7 +19,6 @@
 package org.apache.uima.ducc.rm.scheduler;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -1508,126 +1507,126 @@ class NodePool
 return null;// found 
nothing, heck
 }
 
-private void doEvictions(int[] neededByOrder, HashMapInteger, 
HashMapIRmJob, IRmJob candidates, boolean force)
-{
-
-for ( int nbo = maxorder; nbo  0; nbo-- ) {
-
-if ( neededByOrder[nbo] == 0 ) {  
// these are N-shares
-continue;
-}
-for ( int oo = maxorder; oo  0; oo-- ) {
-HashMapIRmJob, IRmJob jobs = candidates.get(oo);
-if ( jobs == null ) {
-continue;
-}
-
-IteratorIRmJob iter = jobs.values().iterator(); 
// he has something to give.  is it enough?
-while ( iter.hasNext()  (neededByOrder[nbo]  0) ) {
-IRmJob j = iter.next();
-int loss = 0;
-
-switch ( evictionPolicy ) {
-case SHRINK_BY_MACHINE:
-// minimize fragmentation
-loss = 
j.shrinkByOrderByMachine(neededByOrder[nbo], nbo, force, this); // pass in 
number of N-shares of given order that we want
-   
// returns number of quantum shares it had to 
relinquish 
-break;
-case SHRINK_BY_INVESTMENT: 
-// minimize lost work
-loss = j.shrinkByInvestment(neededByOrder[nbo], 
nbo, force, this);// pass in number of N-shares of given order that we want
-   
// returns number of quantum shares it had to 
relinquish 
-break;
-}
+// private void doEvictions(int[] neededByOrder, HashMapInteger, 
HashMapIRmJob, IRmJob candidates, boolean force)
+// {
 
-neededByOrder[nbo]   -= loss;
-neededByOrder[0] -= loss;
-nPendingByOrder[oo]  += loss;
+// for ( int nbo = maxorder; nbo  0; nbo-- ) {
 
-if ( j.countNShares() == 0 ) {
// nothing left? don't look here any more
-iter.remove();
-}
-}
-
-}
-}
-}
-
-/**
- * Here we tell the NP how much we need cleared up.  It will look around 
and try to do that.
- * @deprecated No longer used, the doEvictions code in NodepoolScheduler 
handles evictions by itself.
- * Keeping this for a while for reference.  UIMA-4275
- */
-void doEvictionsByMachine(int [] neededByOrder, boolean force)
-{
-   String methodName = doEvictions;
-//
-// Collect losers that are also squatters, by order, and try them first
-//
-String type;
-type = force ? forced : natural;
-
-logger.debug(methodName, null, getId(),  NeededByOrder, type, on 
entrance eviction, Arrays.toString(neededByOrder));
-
-for ( NodePool np : getChildrenDescending() ) {
-logger.info(methodName, null, Recurse to, np.getId(), from, 
getId(), force:, force);
-np.doEvictionsByMachine(neededByOrder, force);
-logger.info(methodName, null, Recurse from, np.getId(), proceed 
with logic for, getId(), force, force);
-}
-
-// 
-// Adjust neededByOrder to reflect the number of shares that need to 
be preempted by subtracting the
-// number of shares that already are free

svn commit: r1691258 - in /uima/sandbox/uima-ducc/trunk: ./ src/main/admin/ src/main/assembly/ src/main/resources/ uima-ducc-parent/

[challngr]

Author: challngr
Date: Wed Jul 15 19:02:33 2015
New Revision: 1691258

URL: http://svn.apache.org/r1691258
Log:
UIMA-4358 Remove derby and uima-ducc-db project from build.

Modified:
uima/sandbox/uima-ducc/trunk/pom.xml
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_props_manager
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml
uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties
uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml
uima/sandbox/uima-ducc/trunk/uima-ducc-parent/pom.xml

Modified: uima/sandbox/uima-ducc/trunk/pom.xml
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/pom.xml?rev=1691258r1=1691257r2=1691258view=diff
==
--- uima/sandbox/uima-ducc/trunk/pom.xml (original)
+++ uima/sandbox/uima-ducc/trunk/pom.xml Wed Jul 15 19:02:33 2015
@@ -121,11 +121,6 @@
/dependency
dependency
groupIdorg.apache.uima/groupId
-   artifactIduima-ducc-db/artifactId
-   version${project.version}/version
-   /dependency
-   dependency
-   groupIdorg.apache.uima/groupId
artifactIduima-ducc-user/artifactId
version${project.version}/version
/dependency
@@ -180,10 +175,6 @@
/dependency
dependency
groupIdorg.apache.uima/groupId
-   artifactIduima-ducc-db/artifactId
-   /dependency
-   dependency
-   groupIdorg.apache.uima/groupId
artifactIduima-ducc-user/artifactId
/dependency
dependency
@@ -494,7 +485,6 @@
moduleuima-ducc-duccdocs/module
moduleuima-ducc-web/module
moduleuima-ducc-examples/module
-   moduleuima-ducc-db/module
moduleuima-ducc-spawn/module
moduleuima-ducc-user/module
!-- Note: uima-ducc-container has a test dependency on 
uima-ducc-user so must be built after it --

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1691258r1=1691257r2=1691258view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Wed Jul 15 19:02:33 2015
@@ -161,13 +161,7 @@ class Ducc(DuccUtil):
 if ( c == 'pm' ):
 if ( self.pm_jvm_args != None ):
 jvm_opts.append(self.pm_jvm_args)
-
-if ( c == 'db' ):
-if ( self.db_jvm_args != None ):
-jvm_opts.append(self.db_jvm_args)
-self.add_to_classpath(ducc_home + '/lib/derby/*')
-
-
+
 if ( c == 'sm' ):
 if ( self.sm_jvm_args != None ):
 jvm_opts.append(self.sm_jvm_args)

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_props_manager
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_props_manager?rev=1691258r1=1691257r2=1691258view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_props_manager (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_props_manager Wed Jul 15 
19:02:33 2015
@@ -28,7 +28,7 @@ from ducc_util import DuccUtil
 from properties import Properties
 from properties import Property
 
-class DuccPropManager():
+class DuccPropManager:
 def __init__(self):
 # simple bootstratp to establish DUCC_HOME and to set the python path 
so it can
 # find the common code in DUCC_HOME/admin

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1691258r1=1691257r2=1691258view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Wed Jul 15 
19:02:33 2015
@@ -262,7 +262,7 @@ class DuccUtil(DuccBase):
 # CLASSPATH = CLASSPATH + ':' + DH  + 'resources'  UIMA-4168 Use API, 
not classpath to configure log4j
 
 # more are added to some components in ducc.py, e.g.
-#derby, apache-activemq/lib/optional, jetty from ws lib, jsp, 
http- client
+#apache-activemq/lib/optional, jetty from ws lib, jsp, http

svn commit: r1690142 - /uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/main/DuccRmAdmin.java

[challngr]

Author: challngr
Date: Thu Jul  9 18:55:22 2015
New Revision: 1690142

URL: http://svn.apache.org/r1690142
Log:
UIMA-4358 Disable RM admin logging because it breaks qoccupancy and qload 
scripts.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/main/DuccRmAdmin.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/main/DuccRmAdmin.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/main/DuccRmAdmin.java?rev=1690142r1=1690141r2=1690142view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/main/DuccRmAdmin.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/main/DuccRmAdmin.java
 Thu Jul  9 18:55:22 2015
@@ -28,6 +28,7 @@ import org.apache.camel.RuntimeExchangeE
 import org.apache.camel.dataformat.xstream.XStreamDataFormat;
 import org.apache.camel.impl.DefaultCamelContext;
 import org.apache.camel.impl.DefaultClassResolver;
+import org.apache.log4j.Level;
 import org.apache.uima.ducc.common.admin.event.DuccAdminEvent;
 import org.apache.uima.ducc.common.admin.event.RmAdminQLoad;
 import org.apache.uima.ducc.common.admin.event.RmAdminQLoadReply;
@@ -146,7 +147,10 @@ public class DuccRmAdmin
  */
 public DuccLogger getLogger()
 {
-return new DuccLogger(Admin);
+DuccLogger ret = new DuccLogger(admin);
+ret.setLevel(Level.OFF);  // jrc UIMA-4358 disable logging for RM 
admin because
+  // scripting has to scrape stdout and 
the log gets in the way
+return ret;
 }
 
 /**

svn commit: r1687777 - in /uima/sandbox/uima-ducc/trunk: src/main/resources/default.ducc.properties uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex

[challngr]

Author: challngr
Date: Fri Jun 26 14:20:04 2015
New Revision: 168

URL: http://svn.apache.org/r168
Log:
UIMA-4109 Merge doc updates for 2.0.0 function.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex

Modified: 
uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties?rev=168r1=1687776r2=168view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties 
(original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties Fri 
Jun 26 14:20:04 2015
@@ -493,9 +493,15 @@ ducc.uima-as.dd2spring.xsl.path=${DUCC_H
 # maximum amount of time to wait for a response from the JD. This value
 # is used by the JP when sending requests to the JD. 
 ducc.process.request.timeout=3
-# define process container class for DD jobs to instantiate and invoke via 
reflection
+
+# Define process container class for DD jobs to instantiate and invoke via 
reflection. 
+# The container provides classpath  isolation for user defined analytics.
+# The container is instantiated with classes from a System classloader.
 ducc.process.uima.as.container.class = 
org.apache.uima.ducc.user.jp.UimaASProcessContainer
-# define process container class for non-DD jobs to instantiate and invoke via 
reflection
+
+# Define process container class for non-DD jobs to instantiate and invoke via 
reflection. 
+# The container provides classpath  isolation for user defined analytics.
+# The container is instantiated with classes from a System classloader.
 ducc.process.uima.container.class = 
org.apache.uima.ducc.user.jp.UimaProcessContainer
 
 # define the sleep time for JP to use when JD sends empty CAS. In this case the

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex?rev=168r1=1687776r2=168view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex
 Fri Jun 26 14:20:04 2015
@@ -84,6 +84,40 @@
\item[Type] Local
  \end{description}
 
+   \item[ducc.authentication.users.include] \hfill \\
+  Specify users allowed to log in to the web server.  This is used only
+  if {\em ducc.authentication.implementor} is the 
LinuxAuthenticationManager.
+ \begin{description}
+   \item[Default] All users may log in.
+   \item[Type] Local
+ \end{description}
+
+   \item[ducc.authentication.users.exclude] \hfill \\
+  Specify users not allowed to log in to the webserver.  This is used 
only
+  if {\em ducc.authentication.implementor} is the 
LinuxAuthenticationManager.
+ \begin{description}
+   \item[Default] No users are excluded.
+   \item[Type] Local
+ \end{description}
+
+   \item[ducc.authentication.groups.include] \hfill \\
+ Specify groups allowed to log in.  Groups are defined by Unix 
authentication.  Only
+ users in the groups specified here may log in to the web server.  
This is used only
+  if {\em ducc.authentication.implementor} is the 
LinuxAuthenticationManager.
+ \begin{description}
+   \item[Default] Users in all groups may log in.
+   \item[Type] Local
+ \end{description}
+
+   \item[ducc.authentication.groups.exclude] \hfill \\
+ Specify groups not allowed to log in.  Groups are defined by Unix 
authentication. 
+ Users in the groups specified here may not log in to the web server.  
This is used only
+  if {\em ducc.authentication.implementor} is the 
LinuxAuthenticationManager.
+ \begin{description}
+   \item[Default] No users are excluded due to group membership.
+   \item[Type] Local
+ \end{description}
+
\item[ducc.admin.endpoint] \hfill \\
  This is the JMS endpoint name used for DUCC administration messages. 
  \begin{description}
@@ -324,6 +358,21 @@
\item[Type] Local 
  \end{description}
 
+   \item[ducc.driver.jetty.max.threads] \hfill \\
+ Max number of threads in Jetty thread pool servicing incoming  HTTP 
requests. 
+ \begin{description}
+   \item[Default] 100
+   \item[Type] Tuning
+ \end{description}
+
+   \item[ducc.driver.jetty.thread.idletime

svn commit: r1687773 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook: installation.tex logs.tex part4/admin/ducc-users.tex part4/install.tex

[challngr]

Author: challngr
Date: Fri Jun 26 13:23:29 2015
New Revision: 1687773

URL: http://svn.apache.org/r1687773
Log:
UIMA-4109 Conditionally skip hyperref from standalone docs.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/installation.tex

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/logs.tex

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-users.tex

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/installation.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/installation.tex?rev=1687773r1=1687772r2=1687773view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/installation.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/installation.tex
 Fri Jun 26 13:23:29 2015
@@ -18,6 +18,8 @@
 % 
 \documentclass[letterpaper]{article}
 
+\def\DUCCSTANDALONE{}
+
 % space between paragraphs
 \usepackage{parskip}
 

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/logs.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/logs.tex?rev=1687773r1=1687772r2=1687773view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/logs.tex 
(original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/logs.tex 
Fri Jun 26 13:23:29 2015
@@ -18,6 +18,8 @@
 % 
 \documentclass[letterpaper]{article}
 
+\def\DUCCSTANDALONE{}
+
 % space between paragraphs
 \usepackage{parskip}
 

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-users.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-users.tex?rev=1687773r1=1687772r2=1687773view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-users.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-users.tex
 Fri Jun 26 13:23:29 2015
@@ -61,7 +61,7 @@ User mary{ max-allotment = 1000
 User antoinette  { max-allotment = 720  }
 \end{verbatim}
   \caption{Sample User Registration}
-  \label{fig:class.configuration}
+  \label{fig:user.configuration}
 \end{figure}
 
 

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex?rev=1687773r1=1687772r2=1687773view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex
 Fri Jun 26 13:23:29 2015
@@ -321,7 +321,13 @@ $HOME/ducc/logs/job-id
   
 
 \section{Add additional nodes to the DUCC cluster}
-   Additional nodes must meet all \hyperref[sec:install.prerequisites]{\em 
prerequisites}.
+   Additional nodes must meet all 
+   \ifdefined\DUCCSTANDALONE
+   {\em prerequisites} (listed above).
+   \else
+   \hyperref[sec:install.prerequisites]{\em prerequisites}.
+   \fi
+
If user's home directories are on local filesystems the location for user 
logfiles
should be specified to be on a shared filesystem. 
 
@@ -399,7 +405,12 @@ privileged operations.
 
 If a different location is chosen for ducc\_ling the new path needs to be 
specified 
 for ducc.agent.launcher.ducc\_spawn\_path in 
\$DUCC\_HOME/resources/site.ducc.properties.
-See more info at see \hyperref[sec:admin.properties-merge] {Properties 
merging}. 
+See more info at see 
+\ifdefined\DUCCSTANDALONE
+{\em properties merging} in the duccbook. 
+\else
+\hyperref[sec:admin.properties-merge] {Properties merging}. 
+\fi
 
 
 \section{CGroups Installation and Configuration}
@@ -420,8 +431,12 @@ See more info at see \hyperref[sec:admin
 
 To install and configure CGroups for DUCC:
 \begin{enumerate}
-   \item Install the appropriate 
\hyperref[sec:install.prerequisites]{libcgroup package} at level 0.37
- or above.
+   \item Install the appropriate 
+   \ifdefined\DUCCSTANDALONE
+   libcgroup package at level 0.37 or above (see {\em Installation 
Prerequisites}).
+   \else
+   \hyperref[sec:install.prerequisites]{libcgroup package} at level 0.37 or 
above.
+   \fi
 
\item Configure /etc/cgconfig.conf as follows

svn commit: r1686074 - /uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/stop_sim

[challngr]

Author: challngr
Date: Wed Jun 17 16:42:48 2015
New Revision: 1686074

URL: http://svn.apache.org/r1686074
Log:
UIMA-4358 Fix spurious message if 'mesg n' if set in .bashrc.

Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/stop_sim

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/stop_sim
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/stop_sim?rev=1686074r1=1686073r2=1686074view=diff
==
--- uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/stop_sim 
(original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/stop_sim 
Wed Jun 17 16:42:48 2015
@@ -48,7 +48,7 @@ class StopSim(DuccUtil):
 cmp = 'agent'
 else:
 cmp = inst
-self.ssh(node, False, 'kill', signal, pid)
+self.ssh(node, True, 'kill', signal, pid)
 
 return msgs
 

svn commit: r1685849 - /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-services.tex

[challngr]

Author: challngr
Date: Tue Jun 16 14:36:24 2015
New Revision: 1685849

URL: http://svn.apache.org/r1685849
Log:
UIMA-4358 Document broker-jmx-port=none for service registrations.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-services.tex

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-services.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-services.tex?rev=1685849r1=1685848r2=1685849view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-services.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-services.tex
 Tue Jun 16 14:36:24 2015
@@ -273,7 +273,7 @@ ducc.submit.environment.propagated
 The build-in default UIMA-AS pinger supports an argument string of the 
following form
 (with NO embedded blanks):
 \begin{verbatim}
- queue_threshold=nn,window=mm,broker-jmx-port=,meta-timeout=
+ service_ping_arguments=broker-jmx-port=,meta-timeout=
 \end{verbatim}
 
 The keywords in the string have the following meaning:
@@ -281,6 +281,13 @@ ducc.submit.environment.propagated
   \item[broker-jmx-port=] This is the JMX port for the service's 
broker.  If not
 specified, the default of 1099 is used.  This is used to gather 
ActiveMQ statistics
 for the service.
+
+Sometimes it is necessary to disable the gathering of ActiveMQ 
statistics through
+JMX; for example, if the queue is accessed via HTTP instead of 
TCP.  To disable
+JMX statistics, specify the port as ``none''.
+\begin{verbatim}
+service_ping_arguments=broker-jmx-port=none
+\end{verbatim}
   \item[meta-timeout=] This is the time, in milliseconds, to wait 
for a response
 to UIMA-AS {\em get-meta}.  If not specified, the default is 5000 
milliseconds.
 \end{description}

svn commit: r1685880 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main: java/org/apache/uima/ducc/test/randomsleep/FixedSleepAE.java java/org/apache/uima/ducc/test/randomsleep/FixedSleepC

[challngr]

Author: challngr
Date: Tue Jun 16 17:52:32 2015
New Revision: 1685880

URL: http://svn.apache.org/r1685880
Log:
UIMA-4358 Cleanup tests - remove no-longer-used logging.  Correct help text in 
runducc.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/randomsleep/FixedSleepAE.java

uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/randomsleep/FixedSleepCR.java
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/runducc

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/randomsleep/FixedSleepAE.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/randomsleep/FixedSleepAE.java?rev=1685880r1=1685879r2=1685880view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/randomsleep/FixedSleepAE.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/randomsleep/FixedSleepAE.java
 Tue Jun 16 17:52:32 2015
@@ -21,7 +21,6 @@ package org.apache.uima.ducc.test.random
 
 import java.io.File;
 import java.io.FileNotFoundException;
-import java.io.IOException;
 import java.io.PrintWriter;
 import java.lang.management.ManagementFactory;
 import java.lang.management.RuntimeMXBean;
@@ -51,7 +50,6 @@ public class FixedSleepAE extends CasAnn
 Random r;
 Logger logger;
 static boolean initComplete = false;
-Marker marker;
 String AE_Identifier = *^ AE ;
 
 ArrayList long[]  bloated_space = new ArrayList long[] ();
@@ -91,34 +89,6 @@ public class FixedSleepAE extends CasAnn
 }
 }
 
-if ( System.getenv( FAST_INIT_FAIL ) != null ) {
-// we want just enough init to get at least one process into 
RUNNING state, but all
-// subsequent initializations to fail under this scenario.
-String jobid = System.getProperty(ducc.job.id);
-String wd = System.getProperty(user.dir);
-String markerdir = wd + / + jobid + .output;
-System.out.println(LOOK IN  + markerdir);
-File marker = new File(markerdir);
-
-String[] outputs = marker.list();
-int count = 0;
-for ( String s : outputs) {
-if ( s.endsWith(.die) ) count++;
-System.out.println(COUNT  + s +  count =  + count);
-if ( count  2 ) {
-throw new IllegalStateException(foo foo and foo);
-}
-}
-File init_death = new File(markerdir + / + tid + .die);
-System.out.println(CREATE  + init_death);
-try {
-init_death.createNewFile();
-} catch (IOException e1) {
-// TODO Auto-generated catch block
-e1.printStackTrace();
-}
-}
-
 long sleep;
 if ( !initComplete ) {// longer 
init only the first tim
 initComplete = true;
@@ -342,9 +312,6 @@ public class FixedSleepAE extends CasAnn
 String s = sb.toString();
 System.out.println(FROM PRINTLN:  + s);
 logger.log(Level.INFO, FROM LOGGER: + s);
-if ( marker != null ) {
-marker.write(s);
-}
 }
 
 public void destroy()
@@ -355,11 +322,6 @@ public class FixedSleepAE extends CasAnn
 Thread.sleep(3000); // simulate actual 
work being done here
 } catch (InterruptedException e) {
 }
-//  check if process() has been called. If not, marker is still 
null
-if ( marker != null ) {
-marker.flush();
-marker.close();
-}
 System.out.println(AE_Identifier +  Destroy exits);
 }
 
@@ -398,14 +360,6 @@ public class FixedSleepAE extends CasAnn
 System.exit(1);
 }
 
-if ( marker == null) {
-if  (!logid.equals(None) ) {
-marker = new Marker(logid, pid, tid);
-} 
-} else {
-marker.flush();
-}
-
 try{ 
 dolog(msgheader +  sleeping  + elapsed +  MS.);
 String bloat = System.getenv(PROCESS_BLOAT);
@@ -434,6 +388,9 @@ public class FixedSleepAE extends CasAnn
 
 }
 
+//
+// Not used any more.  Kept in src in case we want to resurrect it.
+//
 class Marker
 {
 PrintWriter writer = null;

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/randomsleep/FixedSleepCR.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/randomsleep/FixedSleepCR.java?rev=1685880r1=1685879r2

svn commit: r1685899 - in /uima/sandbox/uima-ducc/trunk: uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/ uima-ducc-common/src/test/resources/node_configuration_data/test1/resources/ uima-du

[challngr]

Author: challngr
Date: Tue Jun 16 19:57:23 2015
New Revision: 1685899

URL: http://svn.apache.org/r1685899
Log:
UIMA-4358 Update CLI tests for changed CLI.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/ClassSeparation.java

uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/ManagedReserveAndCancel.java

uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/ReserveAndCancel.java

uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/ServiceTester.java

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/resources/node_configuration_data/test1/resources/ducc.classes

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/resources/node_configuration_data/test10/resources/ducc.classes

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/resources/node_configuration_data/test11/resources/ducc.classes

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/resources/node_configuration_data/test12/resources/ducc.classes

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/resources/node_configuration_data/test2/resources/ducc.classes

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/resources/node_configuration_data/test3/resources/ducc.classes

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/resources/node_configuration_data/test4/resources/ducc.classes

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/resources/node_configuration_data/test5/resources/ducc.classes

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/resources/node_configuration_data/test6/resources/ducc.classes

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/resources/node_configuration_data/test7/resources/ducc.classes

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/resources/node_configuration_data/test8/resources/ducc.classes

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/resources/node_configuration_data/test9/resources/ducc.classes

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/ClassSeparation.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/ClassSeparation.java?rev=1685899r1=1685898r2=1685899view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/ClassSeparation.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/ClassSeparation.java
 Tue Jun 16 19:57:23 2015
@@ -123,21 +123,16 @@ public class ClassSeparation
 String resid = null;
 
 reserve_props.setProperty(description, Reserve And Cancel);
-reserve_props.setProperty(instance_memory_size, 4);
-reserve_props.setProperty(number_of_instances, 2);
+reserve_props.setProperty(memory_size, 4);
 reserve_props.setProperty(scheduling_class, fixed);
+reserve_props.setProperty(wait_for_completion, true);
 try {
 reserve = new DuccReservationSubmit(reserve_props);
 if ( reserve.execute() ) {
 resid =  + reserve.getDuccId();
-success(testid, Reservation, resid, successful, rc =, 
+reserve.getReturnCode(), :, reserve.getHostsAsString());
-String[] hosts = reserve.getHosts();
-System.out.println( + hosts.length +  hosts assigned);
-if ( hosts.length  0 ) {
-for ( String h : reserve.getHosts() ) {
-System.out.println(+ h);
-}
-}
+String host = reserve.getHost();
+success(testid, Reservation, resid, successful, rc =, 
+reserve.getReturnCode(), :, host);
+System.out.println(+ host);  
 } else {
 fail(testid, Reservation failed, rc =  + 
reserve.getReturnCode());
 }

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/ManagedReserveAndCancel.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/ManagedReserveAndCancel.java?rev=1685899r1=1685898r2=1685899view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/ManagedReserveAndCancel.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/ManagedReserveAndCancel.java
 Tue Jun 16 19:57:23 2015
@@ -75,6 +75,7 @@ public class ManagedReserveAndCancel
 
 reserve_props.setProperty(description, Managed Reserve And Cancel

svn commit: r1685648 - /uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java

[challngr]

Author: challngr
Date: Mon Jun 15 18:01:21 2015
New Revision: 1685648

URL: http://svn.apache.org/r1685648
Log:
UIMA-4358 Misplaced 'return' allowed UIMA-AS jobs in FIXED_SHARE class to be 
evicted under some circumstances.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java?rev=1685648r1=1685647r2=1685648view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java
 Mon Jun 15 18:01:21 2015
@@ -1252,7 +1252,7 @@ public class NodepoolScheduler
  */
 void howMuchFixed(ArrayListResourceClass rcs)
 {
-   String methodName = howMuchFixedShare;
+   String methodName = howMuchFixed;
 
 if ( logger.isTrace() ) {
 logger.info(methodName, null, Scheduling FIXED SHARE for these 
classes:);
@@ -1321,11 +1321,11 @@ public class NodepoolScheduler
 logger.info(methodName, j.getId(), Deferring, no machines 
big enough for the request. NP, np.getId(), 
 available[, np.countNSharesByOrder(order), 
]);
 }
+return;
 } else {
 logger.info(methodName, j.getId(), Nodepool is out of shares: 
NP, np.getId(), 
 available[, np.countNSharesByOrder(order), ]);
 }
-return;
 }
 
 int granted = getAllotmentForJob(j); // in nshares, processes

svn commit: r1684964 - /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/system-logs.tex

[challngr]

Author: challngr
Date: Thu Jun 11 19:14:59 2015
New Revision: 1684964

URL: http://svn.apache.org/r1684964
Log:
UIMA-4109 Updates for 2.0.0.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/system-logs.tex

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/system-logs.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/system-logs.tex?rev=1684964r1=1684963r2=1684964view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/system-logs.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/system-logs.tex
 Thu Jun 11 19:14:59 2015
@@ -38,7 +38,7 @@
 means that log4j configuration files in the user's classpath will not 
interfere
 with DUCC's logger.
 
-The logs are set to roll after some reaching a given size and the number 
of generations
+The logs are set to roll after reaching a given size and the number of 
generations
 is limited to prevent overrunning disk space.  In general the log level is 
set to
 provide sufficient diagnostic output to resolve most issues.
 
@@ -67,8 +67,7 @@
 each running agent.
 
 The log4j file may be customized for each installation to change the 
format or content of the
-log files, according to the rules defined by log4j itself.  This section 
defines the 
-default configuration.
+log files, according to the rules defined by log4j itself.
 
 The general format of a log message is as follows:
 \begin{verbatim}
@@ -86,8 +85,10 @@
   \item[OR] Orchestrator
   \item[WS] Web Server
   \item[Agent] Agent
-  \item[JD] Job Driver
-  \item[JobProcessComponent] Job process, also known as JP
+  \item[JD] Job Driver.  These logs are written to the log directory 
specified in each
+job submission.
+  \item[JobProcessComponent] Job process, also known as JP.  These 
logs are written to the
+log directory specifid in each job submission.
 \end{description}
   \item[sourceFileName] This is the name of the Java source file from 
which the message is emitted.
   \item[method-name] This is the name of the method in {\em 
sourceFileName} which emitted the message.
@@ -118,6 +119,20 @@
   \item Internal schedule
   \item Published schedule
 \end{itemize}
+
+Most useful messages are emitted under log level INFO but a wealth of 
details can
+be seen by incresing the log level to DEBUG or TRACE.  To do so, 
+edit the file 
+{\em  \ducchome/resources/log4j.xml}
+and change the {\em priority} value to {\em debug} (or {\em trace}) in the 
stanza
+similar to that shown here.  Within about a minute the logger will
+pick up the change and increase its log level.
+\begin{verbatim}
+   category name=org.apache.uima.ducc.sm additivity=true
+ priority value=debug/
+ appender-ref ref=smlog / 
+   /category
+\end{verbatim}
 
 \subsection{Bootstrap Configuration}
The RM summarizes its entire configuration when it starts up and prints it 
to the log to
@@ -152,12 +167,12 @@ RM.ResourceManagerComponent- N/A boot  .
 The first configuration lines show the reading and validation of the node 
and class configuration.  Look
 for the string {\em printNodepool} to find these lines:
 \begin{verbatim}
-RM.Config- N/A printNodepool Nodepool --default--
-RM.Config- N/A printNodepoolSearch Order: 100
-RM.Config- N/A printNodepoolNode File: None
-RM.Config- N/A printNodepool   None
-RM.Config- N/A printNodepoolClasses: background low normal high 
normal-all nightly-test reserve
-RM.Config- N/A printNodepoolSubpools: jobdriver power intel
+RM.Config- N/A printNodepool   Nodepool --default--
+RM.Config- N/A printNodepool  Search Order: 100
+RM.Config- N/A printNodepool  Node File: None
+RM.Config- N/A printNodepool None
+RM.Config- N/A printNodepool  Classes: background low normal high 
normal-all nightly-test reserve
+RM.Config- N/A printNodepool  Subpools: jobdriver power intel
  ...
 \end{verbatim}
 
@@ -166,21 +181,22 @@ RM.Config- N/A printNodepoolSubp
about its environment: the ActiveMQ broker, JVM information, OS 
information, DUCC version, etc.  To fine
this search for the string {\em init  Scheduler}.
 \begin{verbatim}
- init  Scheduler running with share quantum   :  15  GB
- init reserved DRAM   :  0  GB
- init DRAM override   :  0  GB
- init scheduler   :  
org.apache.uima.ducc.rm.scheduler.NodepoolScheduler
-
- init DUCC home   :  
/home/challngr/ducc_runtime

svn commit: r1684740 - /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/system-logs.tex

[challngr]

Author: challngr
Date: Wed Jun 10 19:05:18 2015
New Revision: 1684740

URL: http://svn.apache.org/r1684740
Log:
UIMA-4109 Updates for 2.0.0.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/system-logs.tex

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/system-logs.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/system-logs.tex?rev=1684740r1=1684739r2=1684740view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/system-logs.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/system-logs.tex
 Wed Jun 10 19:05:18 2015
@@ -114,7 +114,7 @@
   \item Calculation of job caps
   \item How-much - fair share 
   \item What-of - host assignment and preemption
-  \item Defrag
+  \item Defragmentation
   \item Internal schedule
   \item Published schedule
 \end{itemize}
@@ -203,7 +203,7 @@ R__7434 Job_Driver System  JobDr
 \begin{verbatim}
 LOGHEADER Nodepool: power Host added: power :  bluej290-18   shares  3 total   
 9:   48128 none
 \end{verbatim}
-where the fields mean (f the field isn't described here, the value is not 
relevant to node arrival):
+where the fields mean (if the field isn't described here, the value is not 
relevant to node arrival):
 \begin{description}
   \item[LOGHEADER] is the log entry header as described above.
   \item[Nodepool:power] The node is added to the ``power'' nodepool

svn commit: r1684738 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4: admin/admin-commands.tex admin/ducc-properties.tex system-logs.tex

[challngr]

Author: challngr
Date: Wed Jun 10 19:04:25 2015
New Revision: 1684738

URL: http://svn.apache.org/r1684738
Log:
UIMA-4109 Updates for 2.0.0.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/system-logs.tex

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex?rev=1684738r1=1684737r2=1684738view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
 Wed Jun 10 19:04:25 2015
@@ -436,7 +436,7 @@ Nodepool power
  

 \subsection{rm\_qoccupancy}
-\label{subsec:admin.rm-qoccupancey}
+\label{subsec:admin.rm-qoccupancy}
 
 \subsubsection{{\em Description:}}
 Rm\_qoccupancy provides a list of all known hosts to the RM, and for each 
host, the following information:

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex?rev=1684738r1=1684737r2=1684738view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex
 Wed Jun 10 19:04:25 2015
@@ -820,6 +820,8 @@
 \end{description} 
 
   \item[ducc.orchestrator.state.publish.rate] \hfill \\
+  \phantomsection\label{itm:props-or.state.publish.rate}
+
 The interval in milliseconds between Orchestrator publications of its 
non-abbreviated  
 state. 
 \begin{description}
@@ -1076,6 +1078,8 @@
   
 
 \item[ducc.rm.prediction.fudge] \hfill \\
+  \phantomsection\label{itm:props-rm.prediction.fudge}
+
   When ducc.rm.prediction is enabled, the known initialization time of 
a job's processes plus 
   some fudge factor is used to predict the number of future 
resources needed. The fudge 
   is specified in milliseconds. 
@@ -1093,6 +1097,8 @@
 
 
 \item[ducc.rm.defragmentation.threshold] \hfill \\
+  \phantomsection\label{itm:props-rm.defragmentation.threshold}
+
   If {\em ducc.rm.defragmentation} is enable, limited defragmentation 
of resources is
   performed by the Resource Manager to create sufficient space to 
schedule work 
   that has insufficient resources (new jobs, for example.).  The term

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/system-logs.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/system-logs.tex?rev=1684738r1=1684737r2=1684738view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/system-logs.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/system-logs.tex
 Wed Jun 10 19:04:25 2015
@@ -20,19 +20,24 @@
  \section{Overview}
 
 This chapter provides an overview of the DUCC process logs and how to 
interpret the
-entries therin.
+entries therein.
 
 Each of the DUCC ``head node'' processes writes a detailed log of its 
operation to
 the directory \ducchome/logs.  The logs are managed by Apache log4j.  All 
logs are
 managed by a single log4j configuration file
 \begin{verbatim}
-DUCC_HOME/resources/log4j.xml
+$DUCC_HOME/resources/log4j.xml
 \end{verbatim}
 
 The DUCC logger is configured to check for updates to the log4j.xml
 configuration file and automatically update without the need to restart 
any of
 the DUCC processes.  The update may take up to 60 seconds to take effect.
 
+The DUCC logger is loaded and configured through the log4j API such that 
other
+log4j configuration files that might be in the classpath are ignored.  
This also
+means that log4j configuration files in the user's classpath will not 
interfere
+with DUCC's logger.
+
 The logs are set to roll after some reaching a given size and the number 
of generations
 is limited to prevent overrunning disk space.  In general the log level

svn commit: r1684259 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4: admin/admin-commands.tex admin/ducc-classes.tex ducc-aguide.tex

[challngr]

Author: challngr
Date: Mon Jun  8 19:55:30 2015
New Revision: 1684259

URL: http://svn.apache.org/r1684259
Log:
UIMA-4109 Updates for 2.0.0.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-classes.tex

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/ducc-aguide.tex

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex?rev=1684259r1=1684258r2=1684259view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
 Mon Jun  8 19:55:30 2015
@@ -475,17 +475,17 @@ Nodepool power
 The following example shows seven hosts, one with a preemptable share in 
the {\em --default--}
 nodepool (on bluej290-5), and one with a non-preemptable share in the {\em 
jobdriver} nodepool.
 \begin{verbatim}
-Node Blacklisted Online StatusNodepool Memory 
Order   Free
-  bluej290-5   False   True up --default--   32505856 
2  0
-J[6006] S[ 189] O[2] II[   0] IR[   0] 
E[False] P[False] F[False] I[False]
+Node Blacklisted Online StatusNodepool Memory Order   Free
+  bluej290-5   False   True up --default--   32505856 2  0
+J[6006] S[ 189] O[2] II[   0] IR[   0] E[False] 
P[False] F[False] I[False]
 
-  bluej290-6   False   True up --default--   32505856 
2  2
-  bluej290-7   False   True up --default--   32505856 
2  2
- bluej291-26   False   True upnightly-test   32505856 
2  2
- bluej291-27   False   True upnightly-test   32505856 
2  2
- bluej293-60   False   True up   intel   32505856 
2  2
- bluej537-73   False   True up   jobdriver   32505856 
2  1
-J[5973] S[   1] O[1] II[   0] IR[   0] 
E[False] P[False] F[ True] I[False]
+  bluej290-6   False   True up --default--   32505856 2  2
+  bluej290-7   False   True up --default--   32505856 2  2
+ bluej291-26   False   True upnightly-test   32505856 2  2
+ bluej291-27   False   True upnightly-test   32505856 2  2
+ bluej293-60   False   True up   intel   32505856 2  2
+ bluej537-73   False   True up   jobdriver   32505856 2  1
+J[5973] S[   1] O[1] II[   0] IR[   0] E[False] 
P[False] F[ True] I[False]
 
 
 \end{verbatim}

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-classes.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-classes.tex?rev=1684259r1=1684258r2=1684259view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-classes.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-classes.tex
 Mon Jun  8 19:55:30 2015
@@ -48,7 +48,9 @@ ducc.classes and is specified by the pro
 and the system will refuse to start if the configuration is incorrect.
 
 NOTE: The administrative command {\em check\_ducc -c} may be used to 
verify and validate
-you class configration before attemping to start DUCC.
+you class configration before attemping to start DUCC.  {\em check\_ducc 
-cv} may be used
+to additionally ``pretty-print'' the ducc.classes configuration to the 
console to revel
+class nesting and inheritance.
 
 For example, the diagram below is an abstract representation of all the 
nodes in a
 system.  There are five nodepools defined:
@@ -258,12 +260,7 @@ ducc.classes and is specified by the pro
 
 Omit this property, or set it to 0 to disable the cap.
 
-  \item[max-machines] RESERVE only.  This specifies the maximum number of 
full machines that
-may be reserved by any single job.
-
-Omit this property, or set it to 0 to disable the cap.
-
-  \item[prediction-fudge] FAIR\_SHARE only. When the scheduler is 
considering expanding the
+g  \item[prediction-fudge] FAIR\_SHARE only. When the scheduler is 
considering expanding the
 number of processes for a job

svn commit: r1684244 - /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex

[challngr]

Author: challngr
Date: Mon Jun  8 18:46:23 2015
New Revision: 1684244

URL: http://svn.apache.org/r1684244
Log:
UIMA-4109 Update docs for 2.0.0 - properties merging.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex?rev=1684244r1=1684243r2=1684244view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
 Mon Jun  8 18:46:23 2015
@@ -543,3 +543,55 @@ Nodepool power
 \end{description}
 

+\subsection{ducc\_properties\_manager}
+\label{sec:cli.ducc-properties-manager}
+
+\paragraph{Description:}
+This CLI is used to manually merge or difference two properties files. 
+
+Normally, the DUCC scripts {\em start\_ducc, check\_ducc,}, and {\em 
rm\_configure} automatically
+merge the file {\em default.ducc.properties} and {\em 
site.ducc.properties} when invoked.  
+
+\paragraph{Usage:}
+\begin{description}
+\item[ducc\_props\_manager --merge file1 --with file2 --to file3]
+  Merge two properties files into one.  Properties added to, or changed 
in, the second file
+  are used to override those in the first file, with the result written to 
the third file.
+\item[ducc\_props\_manager --delta file1 --with file2 --to file3]
+  Compare two properties files and write the differences into a third 
file.  The first file is
+  considered a ``master'' file.  Properties with different values in the 
second file, or which
+  do not occur in the first file, are written into the third file.
+\end{description}
+
+\paragraph{Options:}
+\begin{description}
+\item[$--$merge file1]  
+  In this form, the two files specified in the {\em $--$with} and 
{\em$--$to} fields are merged, with the
+  results placed in $--$file3.  Overrides are flagged with a change 
tag and the date of the merge.
+
+  {\em file1} is considered the ``master'' properties file and is 
usually the unmodified file provided
+  with the DUCC distribution, {\em default.ducc.properties}.
+
+  {\em file2} is considered a set of override or additional properties 
and is usually the site local
+  properties file, {\em site.ducc.properties.}
+
+\item[$--$delta file1]
+  In this form, the two files specified in the {\em $--$with} and 
{\em$--$to} fields are compared, with
+  differences placed in $--$file3.
+
+  {\em file1} is considered the ``master'' properties file and is 
usually the unmodified file provided
+  with the DUCC distribution, {\em default.ducc.properties}.
+  
+  {\em file2} is considered the ``external'' properties file and is 
usually the properties file from
+  an older version of DUCC.  
+
+  Differences are placed in {\em $--$file3} which may be a viable 
first cut at a new {\em site.ducc.properties.}
+
+  \item[$--$with file2] This specifies the properties file to merge 
with the master, or to difference
+with the master properties file.
+
+  \item[$--$to file3] This specifies the file to which the results of 
the merge or delta are written.
+ \end{description}
+
+\paragraph{Notes:}
+None.

svn commit: r1684237 - /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/services.tex

[challngr]

Author: challngr
Date: Mon Jun  8 18:09:43 2015
New Revision: 1684237

URL: http://svn.apache.org/r1684237
Log:
UIMA-4109 Update docs for 2.0.0.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/services.tex

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/services.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/services.tex?rev=1684237r1=1684236r2=1684237view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/services.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/services.tex
 Mon Jun  8 18:09:43 2015
@@ -30,7 +30,7 @@
 Users may supply their own ``pingers'' by supplying a Java class 
that implements
 the pinger API.  This is referred to as a ``custom'' pinger in 
this document. 
 There are a number of service registration options which  allow
-specification and parameterization of customn pingers.
+specification and parametrization of custom pingers.
 
   \end{itemize}
   The pinger API enables the following functions for custom pingers:
@@ -40,7 +40,7 @@
   \item enable and disable service autostart, 
   \item notify the Service Manager of the date of last use of a service, 
   \item notify the Service Manager of the health and availability of a 
service, 
-  \item returns a string for display in the DUCC Web server to show 
relevent service information
+  \item returns a string for display in the DUCC Web server to show 
relevant service information
   \end{itemize}
   
 
@@ -59,14 +59,16 @@
\end{itemize}
 
When work enters the system with a declared dependency on a service, 
one of the following
-   actions is tken:
+   actions is taken:
\begin{itemize}
-  \item If the service is not registered, the work request is 
automatically canceled.
-  \item If the service is not running, the Service Manager attempts to 
start it; the job
+  \item If the service is not registered, the work request is 
automatically canceled (to avoid
+wasting resources on a job that is known cannot succeed.)
+  \item If the service registered but not running, the Service Manager 
attempts to start it; the job
remains queued until the service is started and its pinger reports 
good health.
- \item If the service exists but cannot be started, the work remains 
queued and error
+ \item If the service exists but cannot be started, the remains queued 
and error
  status is shown in the web server.  Once the service is working 
again the
- work is allowed to proceed.
+ work is allowed to proceed.  (Jobs already running are not 
directly affected, unless they
+ also cannot access the service.)
\item If the service processes are running but the pinger reports 
failure contacting the service,
the work remains queued with error status shown in the 
webserver.  Once the service
pinger indicates the service is functional again the work is 
allowed to proceed.
@@ -77,7 +79,7 @@
   DUCC supports two types of services: UIMA-AS and CUSTOM:
   
   \begin{description}
-  \item[UIMA-AS] This is a normal UIMA-AS service. DUCC fully 
supports all aspects of UIMA-AS
+  \item[UIMA-AS] This is a normal UIMA-AS service. DUCC fully supports 
all aspects of UIMA-AS
 services with minimal effort from developers.  A default pinger is 
supplied by DUCC
 for UIMA-AS services.  It is legal to define a custom pinger for a 
UIMA-AS service.
 
@@ -86,9 +88,9 @@
   \end{description}
 
   DUCC also supports services that are not managed by DUCC.  These are 
known as {\em ping-only}
-  services.  The registration for a ping-only service contains only 
keywords needed to 
-  support a pinger.  Ping-only services must be defined as custom 
services; there is no
-  default pinger provided for ping-only services.
+  services.  The registration for a ping-only service contains only 
keywords needed to support a
+  pinger, which communicates with the non-DUCC service.  Ping-only 
services must
+  be defined as custom services; there is no default pinger provided for 
ping-only services.
 
   \section{Service Instance IDs}
   \label{sec:service.service.ids}
@@ -96,7 +98,7 @@
   started, the SM assigns monotonically increasing IDs to each service 
instance, starting
   with ID 0, up the the maximum number of instances started.
 
-  If an instance exits unexpectedly, the SM respawns it (unless a failure 
threshold has been
+  If an instance exits

svn commit: r1684246 - /uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install

[challngr]

Author: challngr
Date: Mon Jun  8 19:13:29 2015
New Revision: 1684246

URL: http://svn.apache.org/r1684246
Log:
UIMA-4358 Fix tupo.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install?rev=1684246r1=1684245r2=1684246view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install Mon Jun  8 
19:13:29 2015
@@ -230,7 +230,7 @@ class PostInstall():
 
 if ( self.ducc_head == None ):
 self.ducc_head = self.localhost
-reply = raw_input('Enter hostname of DUCC hoead[' + self.ducc_head 
+ ']')
+reply = raw_input('Enter hostname of DUCC head[' + self.ducc_head 
+ ']')
 if ( reply != '' ):
 self.ducc_head = reply
 

svn commit: r1684079 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin: admin-commands.tex ducc-classes.tex

[challngr]

Author: challngr
Date: Sun Jun  7 21:39:01 2015
New Revision: 1684079

URL: http://svn.apache.org/r1684079
Log:
UIMA-4109 Documentation updates for 2.0.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-classes.tex

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex?rev=1684079r1=1684078r2=1684079view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
 Sun Jun  7 21:39:01 2015
@@ -192,7 +192,7 @@ stop_ducc -c agent@n1 -c agent@n2
 Stop and restart the rm: 
 \begin{verbatim}
 stop_ducc -c rm 
-start_ducc -c rmc 
+start_ducc -c rm
 \end{verbatim}
 
 Components include: 
@@ -202,6 +202,7 @@ start_ducc -c rmc
   \item[pm] The Process Manager. 
   \item[sm] The Service Manager. 
   \item[ws] The Web Server. 
+  \item[broker] The ActiveMQ broker (only if the broker is 
auto-managed).
   \item[agent\@node] Node Agent on the specified node.
   \end{description}
 

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-classes.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-classes.tex?rev=1684079r1=1684078r2=1684079view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-classes.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-classes.tex
 Sun Jun  7 21:39:01 2015
@@ -225,11 +225,6 @@ ducc.classes and is specified by the pro
 ``false''.  This class is never passed to the scheduler and may not be 
referenced
 by jobs.
 
-  \item[cap] This specifies the largest number of shares that any job in 
this class
-may be assigned.  It may be an absolute number or a percentage.  If 
specified as
-a percentage (i.e. it contains a trailing \%), it specifies a 
percentage of the
-total nodes in the containing nodepool.
-
   \item[debug] FAIR\_SHARE only. This specifies the name of a class to 
substitute
 for jobs submitted for debug.  For example, if class {\em normal} 
specifies
 \begin{verbatim}
@@ -247,19 +242,6 @@ ducc.classes and is specified by the pro
 class to use for reservations (Note that either FIXED\_SHARE or 
RESERVE scheduling policies
 are valid for reservations.)
 
-  \item[enforce] RESERVE only.  If specified, then reservations for this 
class must specify a
-memory size that when converted to shares exactly matches an eligible 
machine.  For example,
-if the share quantum is 15GB, a 15GB reservation will never be honored 
by a 240GB machine; 
-for this machine a memory size between 226 \ 240GB would have to be 
specified.  
-The DUCC Web Server's {\em Machines} page displays the recommended 
request size for every machine.  
-
-If {\em enforce} is not specified, the default is ``true''.
-
-If {\em enforce} is set to false, the scheduler will attempt to match 
the reservation as 
-closely as possible to an existing machine, and if it cannot it will 
use the next largest
-machine available.  Thus, a 15GB reservation {\em might} be satisfied 
with a 240GB machine if
-that is all that is available at the time.
-
   \item[expand-by-doubling] FAIR\_SHARE only.  If ``true'', and the {\em 
initialization-cap} is
 set, then after any process has initialized, the job will expand to 
its maximum allowable
 shares by doubling in size each scheduling cycle.  
@@ -358,7 +340,6 @@ Class reserve-base {
   policy = RESERVE
   nodepool = intel
   priority = 1
-  enforce = true
   abstract = true
   max-machines = 10
 }

svn commit: r1683940 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook: images/ducc-webserver/viz.jpg part2/webserver.tex part2/webserver/viz.tex part4/admin/admin-commands.t

[challngr]

Author: challngr
Date: Sat Jun  6 18:33:49 2015
New Revision: 1683940

URL: http://svn.apache.org/r1683940
Log:
UIMA-4109 Documentation updates for 2.0.

Added:

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-webserver/viz.jpg
   (with props)

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/webserver/viz.tex
Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/webserver.tex

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex

Added: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-webserver/viz.jpg
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-webserver/viz.jpg?rev=1683940view=auto
==
Binary file - no diff available.

Propchange: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-webserver/viz.jpg
--
svn:mime-type = application/octet-stream

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/webserver.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/webserver.tex?rev=1683940r1=1683939r2=1683940view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/webserver.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/webserver.tex
 Sat Jun  6 18:33:49 2015
@@ -162,6 +162,10 @@
   \item DuccBook - This manual. 
   \item Machines - This shows the status of all the ducc worker 
nodes. 
 \end{itemize}
+
+\item[Viz]
+This opens a page with a visualization of the system hosts, 
showing all
+scheduled work in the system.
   \end{description}  
 
   % Create well-known link to this spot for HTML version
@@ -212,3 +216,10 @@
   \HCode{a name='DUCC_WS_SYSTEM'/a}
   \fi
   \input{part2/webserver/system.tex}
+
+  % Create well-known link to this spot for HTML version
+  \ifpdf
+  \else
+  \HCode{a name='DUCC_WS_Viz'/a}
+  \fi
+  \input{part2/webserver/viz.tex}

Added: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/webserver/viz.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/webserver/viz.tex?rev=1683940view=auto
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/webserver/viz.tex
 (added)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/webserver/viz.tex
 Sat Jun  6 18:33:49 2015
@@ -0,0 +1,60 @@
+% 
+% Licensed to the Apache Software Foundation (ASF) under one
+% or more contributor license agreements.  See the NOTICE file
+% distributed with this work for additional information
+% regarding copyright ownership.  The ASF licenses this file
+% to you under the Apache License, Version 2.0 (the
+% License); you may not use this file except in compliance
+% with the License.  You may obtain a copy of the License at
+% 
+%   http://www.apache.org/licenses/LICENSE-2.0
+% 
+% Unless required by applicable law or agreed to in writing,
+% software distributed under the License is distributed on an
+% AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+% KIND, either express or implied.  See the License for the
+% specific language governing permissions and limitations
+% under the License.
+% 
+
+\section{Visualization}
+\label{sec:ws.vizualization}
+   This page shows a visualization of all scheduled work.  Every host is 
represented by a square
+   whose area is proportional to the amount of memory on the host.  If 
work is scheduled to a
+   host, it is represented by a rectangle whose area is proportional to 
the amount of memory
+   that is scheduled for the work.  In a multi-user environment, each 
userid is mapped into 
+   a different color, making it possible to see the usage per-user.
+
+   Hovers are provided to show the real memory size of each host, the 
schedulable memory for
+   each host, and the amount of memory scheduled for each bit of work.
+
+   If multiple allocations are made on a single host for the same job or 
service, the rectangles
+   are combined into a single rectangle, reducing clutter and better 
showing the actual usage
+   of the job (or service).   
+
+   Clicking on any box

svn commit: r1683869 - /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/services.tex

[challngr]

Author: challngr
Date: Fri Jun  5 21:00:50 2015
New Revision: 1683869

URL: http://svn.apache.org/r1683869
Log:
UIMA-4109 Updates for 2.0.0.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/services.tex

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/services.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/services.tex?rev=1683869r1=1683868r2=1683869view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/services.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/services.tex
 Fri Jun  5 21:00:50 2015
@@ -58,15 +58,20 @@
   \item Run service pingers and respond to the pinger API as needed.
\end{itemize}
 
-   DUCC provides a ``fast-fail'' for work which references services.  
Incoming work which
-   references a service is canceled by the system under the following 
conditions:
+   When work enters the system with a declared dependency on a service, 
one of the following
+   actions is tken:
\begin{itemize}
- \item The service is not registered.
- \item The service cannot be started.
- \item The service is started, but the service pinger determines the 
service is not
-   viable.
+  \item If the service is not registered, the work request is 
automatically canceled.
+  \item If the service is not running, the Service Manager attempts to 
start it; the job
+   remains queued until the service is started and its pinger reports 
good health.
+ \item If the service exists but cannot be started, the work remains 
queued and error
+ status is shown in the web server.  Once the service is working 
again the
+ work is allowed to proceed.
+   \item If the service processes are running but the pinger reports 
failure contacting the service,
+   the work remains queued with error status shown in the 
webserver.  Once the service
+   pinger indicates the service is functional again the work is 
allowed to proceed.
\end{itemize}
- 
+
 \section{Service Types.}
 \label{sec:services.types}
   DUCC supports two types of services: UIMA-AS and CUSTOM:
@@ -77,7 +82,7 @@
 for UIMA-AS services.  It is legal to define a custom pinger for a 
UIMA-AS service.
 
   \item[CUSTOM] This is any arbitrary service.  Developers must 
provide a custom pinger
-and declare it in the service registration.
+and declare the pinger in the service registration.
   \end{description}
 
   DUCC also supports services that are not managed by DUCC.  These are 
known as {\em ping-only}
@@ -85,6 +90,25 @@
   support a pinger.  Ping-only services must be defined as custom 
services; there is no
   default pinger provided for ping-only services.
 
+  \section{Service Instance IDs}
+  \label{sec:service.service.ids}
+  DUCC 2.0.0 introduces support for constant service instance IDs.  As a 
service is being
+  started, the SM assigns monotonically increasing IDs to each service 
instance, starting
+  with ID 0, up the the maximum number of instances started.
+
+  If an instance exits unexpectedly, the SM respawns it (unless a failure 
threshold has been
+  exceeded).  The new instance is assigned the same instance ID as the 
instance it replaces.
+  This insures that, for example, instance ``three'' is always started as 
instance ``three'',
+  maintained constant over failures and SM restarts.
+
+  The instance ID is communicated to the process through the environment 
with the key
+  {\tt DUCC\_SERVICE\_INSTANCE}.  This key may also be used in service 
registrations if it
+  is desired to pass the instance ID via parameters of some sort.  For 
example:
+\begin{verbatim}
+service_jvm_args   -DSERVICE_ID=${DUCC_SERVICE_INSTANCE}
+process_executable_args -i ${DUCC_SERVICE_INSTANCE}
+\end{verbatim}
+
   \section{Service References and Endpoints} 
   \label{sec:service.endpoints}
   Services are identified by an entity called a {\em service endpoint}.  
Jobs and other
@@ -160,7 +184,9 @@
established to keep the service alive for a while, in anticipation 
that it will be needed
again soon.  When the keep-alive timer expires, and there are no 
more dependent jobs or
services, the reference-started service is automatically stopped to 
free up its resources
-   for other work.
+   for other work.  The time the service is allowed to remain alive is 
known as its
+   {\em linger} time and can be controlled with the {\em 
service

svn commit: r1683775 - /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/rm.tex

[challngr]

Author: challngr
Date: Fri Jun  5 14:48:54 2015
New Revision: 1683775

URL: http://svn.apache.org/r1683775
Log:
UIMA-4109 Updates for 2.0.0.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/rm.tex

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/rm.tex
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/rm.tex?rev=1683775r1=1683774r2=1683775view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/rm.tex
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/rm.tex
 Fri Jun  5 14:48:54 2015
@@ -26,93 +26,108 @@
 \section{Overview}
 
 The DUCC Resource Manager is responsible for allocating cluster resources 
among the various 
-requests for work in the system. DUCC recognizes several classes of work: 
+requests for work in the system. DUCC recognizes several categories of 
work: 
 
 \begin{description}
 \item[Managed Jobs]
-Managed jobs are Java applications implemented in the UIMA 
framework. 
-and are scaled out by DUCC using UIMA-AS. Managed jobs are 
executed as some 
-number of discrete processes distributed over the cluster 
resources. All processes of all jobs 
-are by definition preemptable; the number of processes is allowed 
to increase and decrease 
-over time in order to provide all users access to the computing 
resources. 
+Managed jobs are Java applications implemented in the UIMA 
framework
+and are scaled out by DUCC as some number of discrete processes.  
Processes which 
+compose managed jobs are always restartable and usually 
preemptable.  Preemption
+occurs as a consequence of enforcing fair-share scheduling 
policies.
+
 \item[Services]
-Services are long-running processes which perform some function on 
behalf of 
-jobs or other services. Most DUCC services are UIMA-AS assigned to 
a non-preemptable 
-resource class, as defined below.
+Services are long-running processes which perform some (common) 
function on behalf of 
+jobs or other services.  Services are scaled out as a set of, from 
the RM point of view,
+unrelated non-preemptable processes.  
 
-\item{Reservations}
+\item[Reservations]
 A reservation provides non-preemptable, persistent, dedicated use 
of a full machine or
 some part of a machine to a specific user.
 
-\item{Arbitrary Processes}
+\item[Arbitrary Processes]
 An {\em arbitrary process} or {\em managed reservation} is any 
process at all, which may
-or may not have anything to do with UIMA.  These processes are 
usually used for services,
-or to launch very large Eclipse work-spaces for debugging.  DUCC 
supports this type of 
-process but is not optimized for it.  These processes are usually 
scheduled to be 
-non-preemptable, occupying either a dedicated machine or some 
portion of a machine.
+or may not have anything to do with UIMA.  These processes are 
typically used to
+run non-UIMA tasks such as application builds, large Eclipse 
workspaces for debugging,
+etc. These processes are usually scheduled as non-preemptable 
allocations,
+occupying either a dedicated machine or some portion of a machine.
 
-  \end{description}
+\end{description}
   
-In order to apportion the cumulative memory resource among requests, the 
Resource Manager 
-defines some minimum unit of memory and allocates machines such that a 
fair number of 
-memory units are awarded to every user of the system. This minimum 
quantity is called a share 
-quantum, or simply, a share. The scheduling goal is to award an equitable 
number of memory 
-shares to every user of the system. 
-
-The Resource Manager awards shares according to a fair share policy. The 
memory shares in a 
-system are divided equally among all the users who have work in the 
system. Once an allocation 
-is assigned to a user, that user's jobs are then also assigned an equal 
number of shares, out of the 
-user's allocation. Finally, the Resource Manager maps the share allotments 
to physical resources. 
-
-To map a share allotment to physical resources, the Resource Manager 
considers the amount of 
-memory that each job declares it requires for each process. That 
per-process memory requirement 
-is translated into the minimum number of collocated quantum shares 
required for the process to 
-run. 
-
+To apportion the cumulative memory resource among requests the Resource 
Manager

svn commit: r1682305 - /uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsPing.java

[challngr]

Author: challngr
Date: Thu May 28 18:25:31 2015
New Revision: 1682305

URL: http://svn.apache.org/r1682305
Log:
UIMA-4358 Must use string, not int to set default CAS heap.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsPing.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsPing.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsPing.java?rev=1682305r1=1682304r2=1682305view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsPing.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsPing.java
 Thu May 28 18:25:31 2015
@@ -179,7 +179,7 @@ public class UimaAsPing
 appCtx.put(UimaAsynchronousEngine.ServerUri, broker);
 appCtx.put(UimaAsynchronousEngine.ENDPOINT, endpoint);
 appCtx.put(UimaAsynchronousEngine.GetMetaTimeout, meta_timeout);
-appCtx.put(UIMAFramework.CAS_INITIAL_HEAP_SIZE, 1000);
+appCtx.put(UIMAFramework.CAS_INITIAL_HEAP_SIZE, 1000);
 
 ResourceInitializationException excp = null;
 gmfail = false;

svn commit: r1679555 - in /uima/sandbox/uima-ducc/trunk: uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/

[challngr]

Author: challngr
Date: Fri May 15 12:43:58 2015
New Revision: 1679555

URL: http://svn.apache.org/r1679555
Log:
UIMA-4358 Serialize service start; don't start until previous instance is 
scheduled.
  Remove debugging code from CLI.
  Set default CAS small in pinger.

Modified:

uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceApi.java

uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsPing.java

uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java

uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceApi.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceApi.java?rev=1679555r1=1679554r2=1679555view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceApi.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceApi.java
 Fri May 15 12:43:58 2015
@@ -996,10 +996,6 @@ public class DuccServiceApi
  */
public static void main(String[] args) 
 {
-System.out.println(System.getProperty(java.vendor));
-System.out.println(System.getProperty(java.version));
-System.out.println(System.getProperty(java.home));
-
 boolean rc = false;
 try {
 switch ( getVerb(args) ) {

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsPing.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsPing.java?rev=1679555r1=1679554r2=1679555view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsPing.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsPing.java
 Fri May 15 12:43:58 2015
@@ -179,6 +179,7 @@ public class UimaAsPing
 appCtx.put(UimaAsynchronousEngine.ServerUri, broker);
 appCtx.put(UimaAsynchronousEngine.ENDPOINT, endpoint);
 appCtx.put(UimaAsynchronousEngine.GetMetaTimeout, meta_timeout);
+appCtx.put(UIMAFramework.CAS_INITIAL_HEAP_SIZE, 1000);
 
 ResourceInitializationException excp = null;
 gmfail = false;

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java?rev=1679555r1=1679554r2=1679555view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
 Fri May 15 12:43:58 2015
@@ -134,6 +134,9 @@ public class ServiceHandler
 ListServiceSet services = serviceStateHandler.getServices();
 for ( ServiceSet sset : services ) {
 sset.bootComplete();
+if ( sset.countImplementors()  0 ) {// if something 
was running, let's make sure all the starts are done
+sset.start();
+}
 }
 }
 

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java?rev=1679555r1=1679554r2=1679555view=diff
==
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
 Fri May 15 12:43:58 2015
@@ -480,7 +480,7 @@ public class ServiceSet
 si.setInstanceId(pending_instances.get(id.getFriendly())); // UIMA-4258
 
 handler.addInstance(this, si);
-pendingImplementors.put(id.getFriendly(), si);
+pendingImplementors.put(id.getFriendly(), si);// remember 
which instances we hear about in current OR publication
 }
 
 /**
@@ -488,13 +488,14 @@ public class ServiceSet
  */
 void bootComplete()
 {
+String methodName = bootComplete;
 //
 // During boot, inactive implementors are removed.  Here we cull the 
implementors list to
 // remove stuff that didn't come

svn commit: r1678377 - in /uima/sandbox/uima-ducc/trunk: src/main/admin/build_duccling src/main/admin/ducc_post_install src/main/resources/default.ducc.properties uima-ducc-common/src/main/java/org/ap

[challngr]

Author: challngr
Date: Fri May  8 15:18:01 2015
New Revision: 1678377

URL: http://svn.apache.org/r1678377
Log:
UIMA-4358 Install ducc_ling in architecture-dependent location.

Added:
uima/sandbox/uima-ducc/trunk/src/main/admin/build_duccling   (with props)

uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/OsArch.java
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties

Added: uima/sandbox/uima-ducc/trunk/src/main/admin/build_duccling
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/build_duccling?rev=1678377view=auto
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/build_duccling (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/build_duccling Fri May  8 
15:18:01 2015
@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+# ---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# License); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ---
+
+import os
+import sys
+import glob
+import shutil
+import subprocess
+
+# simple bootstratp to establish DUCC_HOME and to set the python path so it can
+# find the common code in DUCC_HOME/admin
+# Infer DUCC_HOME from our location - no longer use a (possibly inaccurate) 
environment variable
+me = os.path.abspath(__file__)
+ndx = me.rindex('/')
+ndx = me.rindex('/', 0, ndx)
+DUCC_HOME = me[:ndx]  # split from 0 to ndx
+
+sys.path.append(DUCC_HOME + '/bin')
+from ducc_base import DuccBase
+from properties import Properties
+
+
+def main():
+
+props = Properties()
+props.load('../resources/ducc.properties')
+java = props.get('ducc.jvm')
+print 'Using', java
+
+fn = '../lib/uima-ducc/uima-ducc-common*.jar'
+common_jar = glob.glob(fn)
+
+osarch = 'org.apache.uima.ducc.common.utils.OsArch'
+
+CMD = ' '.join([java, '-cp', common_jar[0], osarch])
+print CMD
+
+proc = subprocess.Popen(CMD, shell=True, bufsize=0, 
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+lines = []
+for  line in proc.stdout:
+lines.append(line.strip())
+proc.wait()
+rc = proc.returncode
+
+if ( rc != 0 ):
+print 'build_duccling: Cannot find java property os.arch. The 
following command failed:'
+print CMD
+sys.exit(1)
+
+arch = lines[0]
+print 'Os Architecture:', arch
+
+if ( not os.path.exists(arch) ):
+os.mkdir(arch)
+
+here = os.getcwd()
+os.chdir(../duccling/src)
+rc = os.system(make clean all)
+if ( rc != 0 ):
+print 'Cannot run make in ../duccling/src.  Insure you have a C 
compiler on this system.'
+sys.exit(1)
+os.chdir(here)
+
+shutil.copyfile(../duccling/src/ducc_ling,  arch + '/ducc_ling')
+os.chmod(arch + '/ducc_ling', 0755)
+
+CMD = ' '.join([arch + '/ducc_ling', '-v'])
+rc = os.system(CMD)
+if ( rc != 0 ):
+print 'Could not run', arch +'ducc_ling -v'
+
+print 'ducc_ling is installed for architecture', arch + '. See the 
installation guide for multi-user setup.'
+
+
+main()

Propchange: uima/sandbox/uima-ducc/trunk/src/main/admin/build_duccling
--
svn:executable = *

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install?rev=1678377r1=1678376r2=1678377view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install Fri May  8 
15:18:01 2015
@@ -385,23 +385,20 @@ class PostInstall():
 os.symlink(duccbook, ws_duccbook)
 print '\nDUCC book installed into webserver root\n'
 
-# Make duccling
-here = os.getcwd()
-os.chdir(../duccling/src)
-os.system(make clean all)
-os.chdir(here)
-shutil.copyfile

svn commit: r1678378 - /uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install

[challngr]

Author: challngr
Date: Fri May  8 15:23:56 2015
New Revision: 1678378

URL: http://svn.apache.org/r1678378
Log:
UIMA-4358 Use absolute path to call build_duccling.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install?rev=1678378r1=1678377r2=1678378view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install Fri May  8 
15:23:56 2015
@@ -391,7 +391,7 @@ class PostInstall():
 self.merge_properties()
 
 # Make duccling
-rc = os.system('build_duccling')
+rc = os.system(self.DUCC_HOME + '/admin/build_duccling')
 if ( rc != 0 ):
 print 'Could not build ducc_ling.  Run the command'
 print '   build_duccling'

svn commit: r1677988 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts: runducc start_sim

[challngr]

Author: challngr
Date: Wed May  6 13:53:34 2015
New Revision: 1677988

URL: http://svn.apache.org/r1677988
Log:
UIMA-4358 Updates for modified ducc_ling checks.

Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/runducc
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/runducc
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/runducc?rev=1677988r1=1677987r2=1677988view=diff
==
--- uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/runducc 
(original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/runducc 
Wed May  6 13:53:34 2015
@@ -545,7 +545,7 @@ class RunDucc(DuccUtil):
 print '   -b, --batchfile file'
 print '   This is the batch file describing the submissions. 
Required.'
 print ''
-print '   -i, --init_time seconds'
+print '   -i, --init_time milliseconds'
 print '   This is the AE initialization minimum time in seconds. 
Default:', self.init_time
 print ''
 print '   --init_timeout minutes'
@@ -562,14 +562,13 @@ class RunDucc(DuccUtil):
 print '   The JP will leak in process() until DUCC (hopefully) 
kills us'
 print ''
 print '   -r, --range seconds'
-print '   This is the AE initializion time range over base in 
seconds. Default:', self.init_range
+print '   This is the AE initializion time range over base in 
milliseconds. Default:', self.init_range
 print '   Init time is -i value + random[0, -rvalue]'
 print ''
 print '   -m, --memory_override mem-in-GB'
 print '   Use this instead of what is in the props file. Default: 
None'
 print ''
 print '   -n, --nmachines_override process_deployments_max'
-print '   Override the preconfigured max machines. Use -1 to fully 
inhibit max machines'
 print ''
 print '   -o, --observe'
 print '   Specifies that we submit in keepalive mode and 
observe(watch) the jobs, creating a dir with outputs. Default:', self.observe
@@ -646,7 +645,7 @@ class RunDucc(DuccUtil):
 self.descriptor_as_file = False
 
 try:
-opts, args  = getopt.getopt(argv, 
'b:d:fi:m:nop:q:r:s:t:u:v:w:x:y:z:?h', ['AE', 'DD', 'file', 'SE=', 'IB=', 
'PB=', 'directory=', 'batchfile=', 'init_time=',
+opts, args  = getopt.getopt(argv, 
'b:d:fi:m:n:op:q:r:s:t:u:v:w:x:y:z:?h', ['AE', 'DD', 'file', 'SE=', 'IB=', 
'PB=', 'directory=', 'batchfile=', 'init_time=',
 
'init_fail_cap=', 'range=', 'memory_override=', 'nmachines=', 
'process_timeout=', 
 
'init_timeout=', 'observe'
 
'jd_uima_log=', 'jp_uima_log=',
@@ -671,7 +670,8 @@ class RunDucc(DuccUtil):
 elif o in ('-m', '--memory_override'):
 self.memory_override = a
 elif o in ('-n', '--nmachines'):
-self.max_machines = int(a)
+self.max_machines = int(a) # force ugly failure if not 
a number
+self.max_machines = a
 elif o in ('-p', '--process_timeout'):
 self.process_timeout = a
 elif o in ('-o', '--observe' ):

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim?rev=1677988r1=1677987r2=1677988view=diff
==
--- uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim 
(original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim 
Wed May  6 13:53:34 2015
@@ -80,7 +80,7 @@ class StartSim(DuccUtil):
 
 if ( not self.verify_jvm() ):
 return
-self.verify_duccling(True)
+self.verify_duccling()
 self.verify_limits()
 
 memory = int(memory) * 1024 * 1024# to GB from KB
@@ -359,7 +359,7 @@ class StartSim(DuccUtil):
 return  
 
 self.set_duccling_version()
-self.verify_duccling(True)
+self.verify_duccling()
 
 if ( os.path.exists('sim.pids') ):
 self.pids.load('sim.pids')

svn commit: r1677872 - /uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml

[challngr]

Author: challngr
Date: Tue May  5 18:13:59 2015
New Revision: 1677872

URL: http://svn.apache.org/r1677872
Log:
UIMA-4358 Fix file permissions on some files.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml

Modified: uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml?rev=1677872r1=1677871r2=1677872view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml Tue May  5 18:13:59 
2015
@@ -274,7 +274,7 @@ under the License.
 includeLICENSE/include
 includeNOTICE/include
   /includes
-  fileMode755/fileMode
+  fileMode644/fileMode
   directoryMode755/directoryMode
 /fileSet
 
@@ -290,7 +290,7 @@ under the License.
 fileSet
   directoryuima-ducc-cli/target/directory
   outputDirectorylib/outputDirectory
-  fileMode755/fileMode
+  fileMode644/fileMode
   directoryMode755/directoryMode
   includes
 includeuima-ducc-cli.jar/include

svn commit: r1677876 - in /uima/sandbox/uima-ducc/trunk/src/main/admin: check_ducc ducc.py ducc_post_install ducc_util.py start_ducc

[challngr]

Author: challngr
Date: Tue May  5 18:27:36 2015
New Revision: 1677876

URL: http://svn.apache.org/r1677876
Log:
UIMA-4358 Remove single-user mode.  Do permission  checking in post-install and 
abort if it looks wrong.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1677876r1=1677875r2=1677876view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Tue May  5 18:27:36 
2015
@@ -44,7 +44,11 @@ class CheckDucc(DuccUtil):
 self.check_clock_skew(checkdate)
 self.verify_jvm()
 self.verify_limits()
-self.verify_duccling(self.single_user)
+(viable, elevated, safe) = self.verify_duccling()
+self.duccling_ok(viable, elevated, safe)
+if ( not safe or not viable ):
+print 'NOTOK ducc_ling is not installed correctly.'
+
 return
 
 def verify_activemq(self):
@@ -103,10 +107,7 @@ class CheckDucc(DuccUtil):
 
 if ( self.kill_signal == None ):
 response = Node health checks return.
-if ( self.single_user ) :
-lines = self.ssh(node, True, self.DUCC_HOME + 
/admin/check_ducc, -s, -x, str(int(time(
-else:
-lines = self.ssh(node, True, self.DUCC_HOME + 
/admin/check_ducc, -x, str(int(time(
+lines = self.ssh(node, True, self.DUCC_HOME + /admin/check_ducc, 
-x, str(int(time(
 while 1:
 line = lines.readline()
 if ( 'signal' in line ):
@@ -171,9 +172,6 @@ class CheckDucc(DuccUtil):
 printthe PID file needs rebuilding.  This option causes the 
file to be rebuilt regardless of
 printchanges.
 print 
-print -s --singleuser
-printBypasses the multi-user ducc_ling checks.
-print 
 print -x localdate
 printValidate the local installation, called via ssh usually. 
The date is the dat on the calling machine.
 print 
@@ -189,7 +187,7 @@ class CheckDucc(DuccUtil):
 def main(self, argv):
 
 try:
-opts, args = getopt.getopt(argv, 'cikn:opqsx:h?v', 
['configuration', 'nodelist=', 'int', 'quit', 'kill', 'pids', 'verbose', 
'nothreading', 'singleuser'])
+opts, args = getopt.getopt(argv, 'cikn:opqx:h?v', 
['configuration', 'nodelist=', 'int', 'quit', 'kill', 'pids', 'verbose', 
'nothreading', ])
 except:
 self.usage(Invalid arguments  + ' '.join(argv))
 
@@ -202,7 +200,6 @@ class CheckDucc(DuccUtil):
 checkdate = 0
 config_only = False
 verbose = False
-self.single_user = False
 
 for ( o, a ) in opts:
 if o in ('-c', '--configuration'):
@@ -226,8 +223,6 @@ class CheckDucc(DuccUtil):
 self.kill_signal = '-KILL'
 elif o in ( '--nothreading' ):
 self.disable_threading()
-elif o in ('-s', '--singleuser' ):
-self.single_user = True
 elif o in ('-p', '--pids'):
 redo_pids = True
 elif o in ('-x'):

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1677876r1=1677875r2=1677876view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Tue May  5 18:27:36 2015
@@ -65,7 +65,7 @@ class Ducc(DuccUtil):
 def add_to_classpath(self, lib):
 os.environ['CLASSPATH'] = os.environ['CLASSPATH'] + : + lib
 
-def run_component(self, component, or_parms, numagents, rmoverride, 
background, nodup, localdate, single_user):
+def run_component(self, component, or_parms, numagents, rmoverride, 
background, nodup, localdate):
 
 if ( component == 'all' ):
 component = 'rm,sm,pm,ws,orchestrator'
@@ -101,14 +101,10 @@ class Ducc(DuccUtil):
 if ( not self.verify_limits() ):
 return
 
-if ( not single_user ) :
-dok = self.verify_duccling(single_user)
-if ( not dok ):
-print 'NOTOK ducc_ling is not set up correctly on 
node', self.localhost
-print dok
-return

svn commit: r1677182 - in /uima/sandbox/uima-ducc/trunk/src/main/admin: check_ducc ducc_util.py

[challngr]

Author: challngr
Date: Fri May  1 16:55:44 2015
New Revision: 1677182

URL: http://svn.apache.org/r1677182
Log:
UIMA-4358 Handle banners and motd stuff in ssh. Handle 'not a tty' noise from 
ssh if mesg n is specified.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1677182r1=1677181r2=1677182view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Fri May  1 16:55:44 
2015
@@ -112,6 +112,12 @@ class CheckDucc(DuccUtil):
 if ( 'signal' in line ):
 response = Node health did not complete:  + line
 self.badnodes.append(node)
+# these next two filter junk if 'mesg' is running in a shell rc
+if ( 'stdin: is not a tty' in line ):
+continue
+if ( 'mesg' in line ):
+continue
+
 if ( not line ):
 break
 line = line.strip()

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1677182r1=1677181r2=1677182view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Fri May  1 
16:55:44 2015
@@ -226,9 +226,9 @@ class DuccUtil(DuccBase):
 cmd = ' '.join(CMD)
 #print 'ssh -o BatchMode=yes -o ConnectTimeout=10', host, cmd
 if ( do_wait ):
-return self.popen('ssh -o BatchMode=yes -o ConnectTimeout=10', 
host, cmd)
+return self.popen('ssh -q -o BatchMode=yes -o ConnectTimeout=10', 
host, cmd)
 else:
-return self.spawn('ssh -o BatchMode=yes -o ConnectTimeout=10', 
host, cmd)
+return self.spawn('ssh -q -o BatchMode=yes -o ConnectTimeout=10', 
host, cmd)
 
 
 def set_classpath(self):

svn commit: r1677215 - in /uima/sandbox/uima-ducc/trunk/src/main/admin: ducc.py ducc_util.py

[challngr]

Author: challngr
Date: Fri May  1 19:18:07 2015
New Revision: 1677215

URL: http://svn.apache.org/r1677215
Log:
UIMA-4358 More ssh cleanup.

Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1677215r1=1677214r2=1677215view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Fri May  1 19:18:07 2015
@@ -261,7 +261,7 @@ class Ducc(DuccUtil):
 or_parms = None
 single_user = False
 nodup = False   # we allow duplicates unless asked not to
-localdate = 0
+localdate = time.time()
 
 try:
opts, args = getopt.getopt(argv, 'bc:d:n:o:sk?v', ['or_parms=', 
'nodup' ])

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1677215r1=1677214r2=1677215view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Fri May  1 
19:18:07 2015
@@ -178,7 +178,7 @@ class DuccUtil(DuccBase):
 print Cannot determine if ActiveMq broker is alive.
 return false
 
-lines = self.popen('ssh', self.broker_host, netstat, '-an')
+lines = self.ssh(self.broker_host, True, netstat, '-an')
 #
 # look for lines like this with the configured port in the 4th token, 
and
 # ending with LISTEN:
@@ -206,8 +206,10 @@ class DuccUtil(DuccBase):
 CMD = CMD + ' ' + broker_name
 CMD = 'JAVA_HOME=' + self.java_home() + ' ' + CMD
 print '', CMD
-self.ssh(broker_host, False, CMD)
-pass
+lines = self.ssh(broker_host, True, CMD)
+for l in lines:
+pass   # throw away junk from ssh
+
 
 def nohup(self, cmd, showpid=True):
 cmd = ' '.join(cmd)
@@ -548,7 +550,9 @@ class DuccUtil(DuccBase):
 return answer
 
 def kill_process(self, node, proc, signal):
-self.ssh(node, False, 'kill', signal, proc[1])
+lines = self.ssh(node, True, 'kill', signal, proc[1])
+for l in lines:
+pass # throw away the noise
 
 def clean_shutdown(self):
 DUCC_JVM_OPTS = ' -Dducc.deploy.configuration=' + self.DUCC_HOME + 
/resources/ducc.properties