Author: degenaro
Date: Thu Jan 3 13:51:01 2019
New Revision: 1850235
URL: http://svn.apache.org/viewvc?rev=1850235&view=rev
Log:
UIMA-5928 DUCC Agent quiesce should wait (forever) for non-fairshare
displatchables to complete before shutting down
RM should honor NodeMetrics report of NodeStatus "UnAvailable" as indication of
node "quiesced"
WS should display Machines page Quiesce status: T/F
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/event/ResourceManagerEventListener.java
uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/state/monitoring/INodeState.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/state/monitoring/NodeState.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/system.machines.jsp
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java?rev=1850235&r1=1850234&r2=1850235&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
Thu Jan 3 13:51:01 2019
@@ -259,6 +259,11 @@ public interface IRmPersistence
public Type type() { return Type.Boolean; }
public boolean isIndex() { return true; }
},
+ Quiesced{
+ public String pname() { return "quiesced"; }
+ public Type type() { return Type.Boolean; }
+ public boolean isIndex() { return true; }
+ },
Reservable{
public String pname() { return "reservable"; }
public Type type() { return Type.Boolean; }
Modified:
uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/event/ResourceManagerEventListener.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/event/ResourceManagerEventListener.java?rev=1850235&r1=1850234&r2=1850235&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/event/ResourceManagerEventListener.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/event/ResourceManagerEventListener.java
Thu Jan 3 13:51:01 2019
@@ -20,7 +20,10 @@ package org.apache.uima.ducc.rm.event;
import org.apache.camel.Body;
import org.apache.uima.ducc.common.ANodeStability;
+import org.apache.uima.ducc.common.Node;
import org.apache.uima.ducc.common.NodeIdentity;
+import org.apache.uima.ducc.common.node.metrics.NodeMetrics;
+import org.apache.uima.ducc.common.node.metrics.NodeMetrics.NodeStatus;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.rm.ResourceManager;
@@ -90,8 +93,24 @@ public class ResourceManagerEventListene
*/
public void onNodeMetricsEvent(@Body NodeMetricsUpdateDuccEvent duccEvent)
throws Exception
{
+ String location = "onNodeMetricsEvent";
+ DuccId jobid = null;
//rm.nodeArrives(duccEvent.getNode());
- nodeStability.nodeArrives(duccEvent.getNode());
+ Node node = duccEvent.getNode();
+ nodeStability.nodeArrives(node);
+ if(node != null) {
+ NodeMetrics nodeMetrics = node.getNodeMetrics();
+ if(nodeMetrics != null) {
+ NodeStatus nodeStatus = nodeMetrics.getNodeStatus();
+ String name = null;
+ NodeIdentity nodeIdentity = node.getNodeIdentity();
+ if(nodeIdentity != null) {
+ name = nodeIdentity.getShortName();
+ }
+ logger.debug(location, jobid, name, nodeStatus.name());
+ }
+
+ }
}
public void onNodeInventoryUpdateEvent(@Body NodeInventoryUpdateDuccEvent
duccEvent) throws Exception {
Modified:
uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java?rev=1850235&r1=1850234&r2=1850235&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
Thu Jan 3 13:51:01 2019
@@ -30,6 +30,8 @@ import java.util.Map;
import org.apache.uima.ducc.common.Node;
import org.apache.uima.ducc.common.NodeIdentity;
+import org.apache.uima.ducc.common.node.metrics.NodeMetrics;
+import org.apache.uima.ducc.common.node.metrics.NodeMetrics.NodeStatus;
import org.apache.uima.ducc.common.persistence.rm.IRmPersistence.RmNodes;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.SystemPropertyResolver;
@@ -61,7 +63,8 @@ class NodePool
HashMap<Integer, HashMap<Node, Machine>> machinesByOrder = new
HashMap<Integer, HashMap<Node, Machine>>(); // All schedulable machines, not
necessarily free
HashMap<String, Machine> machinesByName = new
HashMap<String, Machine>(); // by name, for nodepool support
HashMap<String, Machine> deadByName = new
HashMap<String, Machine>(); // anything we move to offline or
unresponsive,
-
// but with the same name we used, because
+
+ List<Node> quiesceMachines = new ArrayList<Node>();
// sometimes stupid domain gets in the way
HashMap<String, Machine> machinesByIp = new
HashMap<String, Machine>(); // by IP, for nodepool support
@@ -1067,6 +1070,76 @@ class NodePool
}
}
+ // determine if Node has been quiesced
+ boolean isQuiesce(Node node) {
+ String methodName = "isQuiesce";
+ boolean retVal = false;
+ NodeMetrics nm = node.getNodeMetrics();
+ if(nm != null) {
+ NodeStatus ns = nm.getNodeStatus();
+ if(ns != null) {
+ switch(ns) {
+ case UnAvailable:
+ logger.info(methodName, null,
node.getNodeIdentity().getShortName(),"node status = "+ns.name());
+ retVal = true;
+ break;
+ default:
+ logger.debug(methodName, null,
node.getNodeIdentity().getShortName(),"node status = "+ns.name());
+ break;
+ }
+ }
+ else {
+ logger.warn(methodName, null,
node.getNodeIdentity().getShortName(),"node status missing");
+ }
+ }
+ else {
+ logger.warn(methodName, null,
node.getNodeIdentity().getShortName(),"node metrics missing");
+ }
+ return retVal;
+ }
+
+ // process Node that is quiesced
+ private void handle_quiesced(Node node) {
+ String methodName = "handle_quiesced";
+ String name = node.getNodeIdentity().getCanonicalName();
+ if(quiesceMachines.contains(node)) {
+ logger.trace(methodName, null, "Node ", name, " is already
quiesced.");
+ }
+ else {
+ quiesceMachines.add(node);
+ logger.info(methodName, null, "Node ", name, " is
quiesced.");
+ if(allMachines.containsKey(node)) {
+ Machine machine = allMachines.get(node);
+ signalDb(machine, RmNodes.Quiesced, true);
+ logger.info(methodName, null, "Node ", name, "
db marked quiesced.");
+ }
+ else {
+ logger.info(methodName, null, "Node ", name, "
is new.");
+ }
+ }
+ }
+
+ // process Node that is not quiesced
+ private void handle_not_quiesced(Node node) {
+ String methodName = "handle_not_quiesced";
+ String name = node.getNodeIdentity().getCanonicalName();
+ if(!quiesceMachines.contains(node)) {
+ logger.trace(methodName, null, "Node ", name, " is already not
quiesced.");
+ }
+ else {
+ quiesceMachines.remove(node);
+ logger.info(methodName, null, "Node ", name, " is not
quiesced.");
+ if(allMachines.containsKey(node)) {
+ Machine machine = allMachines.get(node);
+ signalDb(machine, RmNodes.Quiesced, false);
+ logger.info(methodName, null, "Node ", name, "
db marked not quiesced.");
+ }
+ else {
+ logger.info(methodName, null, "Node ", name, "
is new.");
+ }
+ }
+ }
+
/**
* Handle a new node update.
*/
@@ -1080,6 +1153,14 @@ class NodePool
String n = node.getNodeIdentity().getCanonicalName();
+ boolean node_quiesced = isQuiesce(node);
+ if(node_quiesced) {
+ handle_quiesced(node);
+ }
+ else {
+ handle_not_quiesced(node);
+ }
+
// if it's offline it can't be restored like this.
if ( offlineMachines.containsKey(node) ) {
Machine m = offlineMachines.get(node);
@@ -1142,6 +1223,7 @@ class NodePool
updated++;
Map<RmNodes, Object> props = initDbProperties(allMachines.get(key));
+ props.put(RmNodes.Quiesced, node_quiesced);
props.put(RmNodes.Responsive, true);
props.put(RmNodes.Online, true);
try {
@@ -1303,8 +1385,11 @@ class NodePool
boolean isSchedulable(Machine m)
{
if ( m.isBlacklisted() ) return false;
+
if ( unresponsiveMachines.containsKey(m.key()) ) return false;
if ( offlineMachines.containsKey(m.key()) ) return false;
+
+ if ( quiesceMachines.contains(m.key()) ) return false;
return true;
}
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java?rev=1850235&r1=1850234&r2=1850235&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
Thu Jan 3 13:51:01 2019
@@ -1818,6 +1818,10 @@ public class DuccHandlerClassic extends
row.append("<td>");
row.append(nodeState.getOnline(machineInfo.getName(), "-"));
row.append("</td>");
+ // Quiesced
+ row.append("<td>");
+ row.append(nodeState.getQuiesced(machineInfo.getName(), "-"));
+ row.append("</td>");
// IP
row.append("<td>");
row.append(machineInfo.getIp());
@@ -2032,6 +2036,10 @@ public class DuccHandlerClassic extends
row.append("<td>");
row.append("");
row.append("</td>");
+ // Quiesced
+ row.append("<td>");
+ row.append("");
+ row.append("</td>");
// IP
row.append("<td>");
row.append("");
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java?rev=1850235&r1=1850234&r2=1850235&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
Thu Jan 3 13:51:01 2019
@@ -1801,6 +1801,8 @@ public class DuccHandlerJsonFormat exten
row.add(new JsonPrimitive(sb.toString()));
// Online
row.add(new
JsonPrimitive(nodeState.getOnline(machineInfo.getName(), "-")));
+ // Quiesced
+ row.add(new
JsonPrimitive(nodeState.getQuiesced(machineInfo.getName(), "-")));
// IP
row.add(new JsonPrimitive(machineInfo.getIp()));
// Name
@@ -2025,6 +2027,8 @@ public class DuccHandlerJsonFormat exten
row.add(new JsonPrimitive("Total"));
// Online
row.add(new JsonPrimitive(""));
+ // Quiesced
+ row.add(new JsonPrimitive(""));
// IP
row.add(new JsonPrimitive(""));
// Name
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/state/monitoring/INodeState.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/state/monitoring/INodeState.java?rev=1850235&r1=1850234&r2=1850235&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/state/monitoring/INodeState.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/state/monitoring/INodeState.java
Thu Jan 3 13:51:01 2019
@@ -20,6 +20,7 @@ package org.apache.uima.ducc.ws.state.mo
public interface INodeState {
public String getOnline(String node, String otherwise);
+ public String getQuiesced(String node, String otherwise);
// admin
public void start();
public void stop();
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/state/monitoring/NodeState.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/state/monitoring/NodeState.java?rev=1850235&r1=1850234&r2=1850235&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/state/monitoring/NodeState.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/state/monitoring/NodeState.java
Thu Jan 3 13:51:01 2019
@@ -50,6 +50,7 @@ public class NodeState implements INodeS
private Monitor monitor = null;
private String key_online = "online";
+ private String key_quiesced = "quiesced";
private NodeState() {
start();
@@ -101,6 +102,38 @@ public class NodeState implements INodeS
logger.debug(location, jobid,
node+"=="+retVal);
break;
}
+ else {
+ logger.debug(location, jobid,
key+"!="+node);
+ }
+ }
+ }
+ else {
+ logger.warn(location, jobid, "size:"+0);
+ }
+ }
+ else {
+ logger.error(location, jobid, "node:"+node);
+ }
+ return retVal;
+ }
+
+ // general function to get quiesced status for node
+ @Override
+ public String getQuiesced(String node, String otherwise) {
+ String location = "getQuiesced";
+ String retVal = otherwise;
+ if(node != null) {
+ if(map.size() > 0) {
+ for(Entry<String, Map<String, Object>> entry :
map.entrySet()) {
+ String key = entry.getKey();
+ if(key.equals(node)) {
+ logger.debug(location, jobid,
key+"=="+node);
+ Map<String, Object> value =
entry.getValue();
+ Boolean value_quiesced =
(Boolean) value.get(key_quiesced);
+ retVal = ""+value_quiesced;
+ logger.debug(location, jobid,
node+"=="+retVal);
+ break;
+ }
else {
logger.debug(location, jobid,
key+"!="+node);
}
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/system.machines.jsp
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/system.machines.jsp?rev=1850235&r1=1850234&r2=1850235&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/system.machines.jsp
(original)
+++ uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/system.machines.jsp
Thu Jan 3 13:51:01 2019
@@ -39,7 +39,6 @@ under the License.
"sAjaxSource":
"ducc-servlet/json-format-aaData-machines",
aaSorting: [],
"fnRowCallback" : function(nRow,aData,iDisplayIndex) {
- $('td:eq(5)' , nRow).css( "text-align", "right" );
$('td:eq(6)' , nRow).css( "text-align", "right" );
$('td:eq(7)' , nRow).css( "text-align", "right" );
$('td:eq(8)' , nRow).css( "text-align", "right" );
@@ -47,6 +46,7 @@ under the License.
$('td:eq(10)' ,
nRow).css( "text-align", "right" );
$('td:eq(11)' ,
nRow).css( "text-align", "right" );
$('td:eq(12)' ,
nRow).css( "text-align", "right" );
+ $('td:eq(13)' ,
nRow).css( "text-align", "right" );
return nRow;
},
} );
@@ -108,6 +108,7 @@ if (table_style.equals("scroll")) {
<tr class="ducc-header">
<th align="left" title="The heartbeat status, as reported to
ducc-mon">Status</th>
<th align="left" title="The online status, as determined by resource
manager">Online</th>
+ <th align="left" title="The quiesce status, as determined by resource
manager">Quiesce</th>
<th align="left" title="The host IP">IP</th>
<th align="left" title="The host name">Name</th>
<th align="left" title="The host node pool">Nodepool</th>
@@ -139,6 +140,7 @@ if (table_style.equals("classic")) {
<tr class="ducc-head">
<th align="left" class="none" title="The heartbeat
status, as reported to ducc-mon">Status</th>
<th align="left" class="none" title="The online
status, as determined by resource manager">Online</th>
+ <th align="left" title="The quiesce status, as determined by
resource manager">Quiesce</th>
<th align="left" class="none" title="The host
IP">IP</th>
<th align="left" class="none" title="The host
name">Name</th>
<th align="left" class="none" title="The host node
pool">Nodepool</th>
