svn commit: r1719195 - in /uima/sandbox/uima-ducc/trunk: src/main/admin/ uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/ uima-ducc-database/src/main/java/org/apache/uima/duc
Author: challngr
Date: Thu Dec 10 20:13:26 2015
New Revision: 1719195
URL: http://svn.apache.org/viewvc?rev=1719195&view=rev
Log:
UIMA-4577 Tweaks so database classes can be used in CLI utilities. Implement
rm_qoccupancy using
the DB as proof of concept.
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmQLoad.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmShareState.java
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy?rev=1719195&r1=1719194&r2=1719195&view=diff
==
--- uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy Thu Dec 10
20:13:26 2015
@@ -21,59 +21,95 @@
import os
import sys
+import glob
import subprocess
from ducc_util import DuccUtil
class DuccRmQOccupancy(DuccUtil):
-def format_shares(self, sh):
-for s in sh:
-if ( s['blacklisted'] ):
-pass
-else:
-print '%19s J[%8d] S[%8d] O[%d] II[%8d] IR[%8d] E[%5s] P[%5s]
F[%5s] I[%5s]' % ('', s['jobid'], s['shareid'], s['order'],
s['investment-init'], s['investment-run'],
- s['evicted'], s['purged'], s['fixed'], s['initialized'])
-
-
-def format_machines(self, lines):
-
-print("%20s %11s %6s %6s %15s %10s %5s %6s" % ("Node", "Blacklisted",
"Online", "Status", "Nodepool", "Memory", "Order", "Free"))
-
-for m in lines:
-print "%20s %11s %6s %6s %15s %10s %5s %6s" % (m['name'],
m['blacklisted'], m['online'], m['status'], m['nodepool'], m['memory'],
m['order'], m['shares-free'])
-if ( len(m['shares']) != 0 ) :
-self.format_shares(m['shares'])
-print ''
+def format(self, nodes, shares):
+print("%20s %11s %6s %6s %15s %10s %6s %6s %6s %8s %7s %10s %8s" %
("Node", "Blacklisted", "Online", "Status", "Nodepool", "Memory", "Order",
"Free", "In-Use", "Np-InUse", "Quantum", "Reservable", "Classes"))
+print("%20s %11s %6s %6s %15s %10s %6s %6s %6s %8s %7s %10s %8s" %
("", "---", "--", "--", "", "--", "-",
"", "--", "", "---", "--", "---"))
+for n in nodes:
+if (n['responsive']):
+status = 'up'
+else:
+status = 'down'
+print "%20s %11s %6s %6s %15s %10s %3s(Q) %6s %6s %8s %7s %10s
%8s" % (n['name'], n['blacklisted'], n['online'], status, n['nodepool'],
+
n['memory'], n['share_order'], n['shares_left'], n['assignments'],
+
n['np_assignments'], n['quantum'], n['reservable'], n['classes'])
+if ( shares.has_key(n['name']) ):
+for s in shares[n['name']]:
+fmt = '%19s ' + s['jobtype'] +'[%8d] S[%8d] O[%d] II[%8d]
IR[%8d] E[%5s] P[%5s] F[%5s] S[%10s]'
+state = s['state']
+if ( state == 'null' ):
+state = "Assigned"
+print fmt % ('', s['job_id'], s['ducc_dbid'],
s['share_order'], s['init_time'], s['investment'], s['evicted'], s['purged'],
s['fixed'], state)
-def format(self, lines):
-qoccupancy = eval(lines)
-self.format_machines(qoccupancy)
+print ''
+# Given DUCC_HOME, a directory, and part of the name of a jar, find the
actual name of the jar which will
+# likeley be versioned
+def resolve_jar(self, DH, dirname, basename):
+partial = DH + '/' + dirname + '/'+ basename + '*'
+ret = glob.glob(partial)
+return ret[0]
+
def main(self, argv):
if ( argv == '-h' or argv == '-?' or argv == '--help' ):
print 'rm_qload queries and formats the current
svn commit: r1719193 - /uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/json/MachineFacts.java
Author: degenaro
Date: Thu Dec 10 20:12:18 2015
New Revision: 1719193
URL: http://svn.apache.org/viewvc?rev=1719193&view=rev
Log:
UIMA-4704 DUCC Web Server (WS) Machines page enhancements when using DB
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/json/MachineFacts.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/json/MachineFacts.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/json/MachineFacts.java?rev=1719193&r1=1719192&r2=1719193&view=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/json/MachineFacts.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/json/MachineFacts.java
Thu Dec 10 20:12:18 2015
@@ -26,10 +26,13 @@ public class MachineFacts implements Ser
private static final long serialVersionUID = 1L;
public String status;
+ public String statusReason;
public String ip;
public String name;
public String memTotal;
public String memFree;
+ public String memReserve;
+ public String quantum;
public String swapInuse;
public String swapDelta;
public String swapFree;
@@ -51,10 +54,13 @@ public class MachineFacts implements Ser
)
{
this.status = status;
+ this.statusReason = "";
this.ip = ip;
this.name = name;
this.memTotal = memTotal;
this.memFree = memFree;
+ this.memReserve = "0";
+ this.quantum = "";
this.swapInuse = swapInuse;
this.swapDelta = swapDelta;
this.swapFree = swapFree;
svn commit: r1719194 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws: ./ server/
Author: degenaro
Date: Thu Dec 10 20:12:41 2015
New Revision: 1719194
URL: http://svn.apache.org/viewvc?rev=1719194&view=rev
Log:
UIMA-4704 DUCC Web Server (WS) Machines page enhancements when using DB
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesDataHelper.java
- copied, changed from r1718861,
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/Helper.java
Removed:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/Helper.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java?rev=1719194&r1=1719193&r2=1719194&view=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
Thu Dec 10 20:12:41 2015
@@ -22,6 +22,7 @@ import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentSkipListMap;
@@ -39,6 +40,8 @@ import org.apache.uima.ducc.common.utils
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.transport.event.NodeMetricsUpdateDuccEvent;
import org.apache.uima.ducc.transport.event.ProcessInfo;
+import org.apache.uima.ducc.ws.db.DbQuery;
+import org.apache.uima.ducc.ws.db.IDbMachine;
import org.apache.uima.ducc.ws.types.Ip;
import org.apache.uima.ducc.ws.types.NodeId;
import org.apache.uima.ducc.ws.types.UserId;
@@ -366,7 +369,22 @@ public class DuccMachinesData {
return retVal;
}
+ public void enhance(MachineFacts facts, Map
dbMachineMap) {
+ if(facts != null) {
+ if(dbMachineMap != null) {
+ String[] machineStatus =
DuccMachinesDataHelper.getMachineStatus(facts, dbMachineMap);
+ facts.status = machineStatus[0];
+ facts.statusReason = machineStatus[1];
+ String reserveSize =
DuccMachinesDataHelper.getMachineReserveSize(facts, dbMachineMap);
+ facts.memReserve = reserveSize;
+ String quantum =
DuccMachinesDataHelper.getMachineQuantum(facts, dbMachineMap);
+ facts.quantum = quantum;
+ }
+ }
+ }
+
public MachineFactsList getMachineFactsList() {
+ Map dbMachineMap =
DbQuery.getInstance().getMapMachines();
MachineFactsList factsList = new MachineFactsList();
ConcurrentSkipListMap sortedMachines =
getSortedMachines();
Iterator iterator;
@@ -385,6 +403,7 @@ public class DuccMachinesData {
List aliens = machineInfo.getAliens();
String heartbeat = ""+machineInfo.getElapsed();
MachineFacts facts = new
MachineFacts(status,ip,name,memTotal,memFree,swapInuse,swapDelta,swapFree,cGroups,aliens,heartbeat);
+ enhance(facts,dbMachineMap);
factsList.add(facts);
}
return factsList;
Copied:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesDataHelper.java
(from r1718861,
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/Helper.java)
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesDataHelper.java?p2=uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesDataHelper.java&p1=uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/Helper.java&r1=1718861&r2=1719194&rev=1719194&view=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/Helper.java
(original)
+++
uima/sandbox/uima-duc
