Author: degenaro
Date: Mon May 16 20:15:37 2016
New Revision: 1744130
URL: http://svn.apache.org/viewvc?rev=1744130&view=rev
Log:
UIMA-4923 DUCC Web Server (WS) should not display nodes that have been removed
from service on Machines page
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccNodes.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/NodeId.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java?rev=1744130&r1=1744129&r2=1744130&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
Mon May 16 20:15:37 2016
@@ -48,6 +48,11 @@ import org.apache.uima.ducc.ws.types.Nod
import org.apache.uima.ducc.ws.types.UserId;
import org.apache.uima.ducc.ws.utils.DatedNodeMetricsUpdateDuccEvent;
+/**
+ * A class to manage information about machines comprising data
+ * from Agents and Resource Manager (RM). The former report
+ * via node metrics publications the later reports via database.
+ */
public class DuccMachinesData {
private static DuccLogger logger =
DuccLoggerComponents.getWsLogger(DuccMachinesData.class.getName());
@@ -70,6 +75,10 @@ public class DuccMachinesData {
private static
ConcurrentSkipListMap<String,TreeMap<String,NodeUsersInfo>>
ipToNodeUsersInfoMap = new
ConcurrentSkipListMap<String,TreeMap<String,NodeUsersInfo>>();
+ private static MachineFactsList machineFactsList = new
MachineFactsList();
+
+ private static DbQuery dbQuery = DbQuery.getInstance();
+
public static DuccMachinesData getInstance() {
return duccMachinesData;
}
@@ -99,6 +108,7 @@ public class DuccMachinesData {
logger.debug(location, jobid, "put: "+value);
}
sortedMachines = map;
+ updateMachineFactsList();
}
catch(Exception e) {
logger.error(location, jobid, e);
@@ -391,7 +401,7 @@ public class DuccMachinesData {
return retVal;
}
- public void enhance(MachineFacts facts, Map<String, IDbMachine>
dbMachineMap) {
+ private void enhance(MachineFacts facts, Map<String, IDbMachine>
dbMachineMap) {
if(facts != null) {
if(dbMachineMap != null) {
String[] machineStatus =
DuccMachinesDataHelper.getMachineStatus(facts, dbMachineMap);
@@ -405,8 +415,35 @@ public class DuccMachinesData {
}
}
- public MachineFactsList getMachineFactsList() {
- Map<String, IDbMachine> dbMachineMap =
DbQuery.getInstance().getMapMachines();
+ /**
+ * Create a cached data set employed by the WS
+ * to display the System -> Machines page. The
+ * code is dual-pathed, depending on whether or
+ * not the system is configured to use database.
+ */
+ private void updateMachineFactsList() {
+ String location = "updateMachineFactsList";
+ try {
+ DbQuery dbQuery = DbQuery.getInstance();
+ if(!dbQuery.isEnabled()) {
+ updateMachineFactsListAgent();
+ }
+ else {
+ updateMachineFactsListDb();
+ }
+ }
+ catch(Exception e) {
+ logger.error(location, jobid, e);
+ System.out.println(e.getMessage());
+ }
+ }
+
+ /**
+ * Create a machines facts list based
+ * in part on the information provided by the Agents, and
+ * in part on the data comprising the ducc.nodes file.
+ */
+ private void updateMachineFactsListAgent() {
MachineFactsList factsList = new MachineFactsList();
ConcurrentSkipListMap<MachineInfo,NodeId> sortedMachines =
getSortedMachines();
Iterator<MachineInfo> iterator;
@@ -426,10 +463,85 @@ public class DuccMachinesData {
List<String> aliens = machineInfo.getAliens();
String heartbeat = ""+machineInfo.getElapsed();
MachineFacts facts = new
MachineFacts(status,ip,name,memTotal,memFree,swapInuse,swapDelta,swapFree,cpu,cGroups,aliens,heartbeat);
- enhance(facts,dbMachineMap);
+ // when not using DB, memResrve == memTotal
+ facts.memReserve = memTotal;
factsList.add(facts);
}
- return factsList;
+ machineFactsList = factsList;
+ }
+
+ /**
+ * Create a machines facts list based
+ * in part on the entries in the RM-maintained database, and
+ * in part on the information provided by the Agents, and
+ * in part on the data comprising the ducc.nodes file.
+ */
+ private void updateMachineFactsListDb() {
+ // The returnable
+ MachineFactsList mfl = new MachineFactsList();
+ // Working map used to generate the returnable
+ ConcurrentSkipListMap<MachineInfo,NodeId> dbSortedMachines =
new ConcurrentSkipListMap<MachineInfo,NodeId>();
+ // Get map from DB courtesy of RM
+ Map<String, IDbMachine> dbMapMachines =
dbQuery.getMapMachines();
+ // Working list of known machines, by short name
+ List<String> knownShortNames = new ArrayList<String>();
+ // Update working map and list of short name from DB
+ for(Entry<String, IDbMachine> entry : dbMapMachines.entrySet())
{
+ String name = entry.getKey();
+ NodeId nodeId = new NodeId(name);
+ MachineInfo mi = unsortedMachines.get(nodeId);
+ if(mi != null) {
+ dbSortedMachines.put(mi, nodeId);
+ String shortName = nodeId.getShortName();
+ knownShortNames.add(shortName);
+ }
+ }
+ // Initialize returnable with "defined" machines
+ ArrayList<String> duccNodes = DuccNodes.getInstance().get();
+ for(String name : duccNodes) {
+ // skip defined machine if it already appears in DB
+ if(knownShortNames.contains(name)) {
+ continue;
+ }
+ String status = "defined";
+ String ip = "";
+ String memTotal = "";
+ String memFree = "";
+ String swapInuse = "";
+ String swapDelta = "";
+ String swapFree = "";
+ double cpu = 0;
+ boolean cGroups = false;
+ List<String> aliens = new ArrayList<String>();
+ String heartbeat = "";
+ MachineFacts facts = new
MachineFacts(status,ip,name,memTotal,memFree,swapInuse,swapDelta,swapFree,cpu,cGroups,aliens,heartbeat);
+ mfl.add(facts);
+ }
+ // Augment returnable with data from Agents & RM (from DB)
+ for(Entry<MachineInfo, NodeId> entry :
dbSortedMachines.entrySet()) {
+ MachineInfo machineInfo = entry.getKey();
+ String status = machineInfo.getStatus();
+ String ip = machineInfo.getIp();
+ String name = machineInfo.getName();
+ String memTotal = machineInfo.getMemTotal();
+ String memFree = machineInfo.getMemFree();
+ String swapInuse = machineInfo.getSwapInuse();
+ String swapDelta = ""+machineInfo.getSwapDelta();
+ String swapFree = machineInfo.getSwapFree();
+ double cpu = machineInfo.getCpu();
+ boolean cGroups = machineInfo.getCgroups();
+ List<String> aliens = machineInfo.getAliens();
+ String heartbeat = ""+machineInfo.getElapsed();
+ MachineFacts facts = new
MachineFacts(status,ip,name,memTotal,memFree,swapInuse,swapDelta,swapFree,cpu,cGroups,aliens,heartbeat);
+ // Add info from Agent that DB does not have
+ enhance(facts, dbMapMachines);
+ mfl.add(facts);
+ }
+ machineFactsList = mfl;
+ }
+
+ public MachineFactsList getMachineFactsList() {
+ return machineFactsList;
}
}
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccNodes.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccNodes.java?rev=1744130&r1=1744129&r2=1744130&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccNodes.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccNodes.java
Mon May 16 20:15:37 2016
@@ -30,6 +30,14 @@ import org.apache.uima.ducc.common.utils
import org.apache.uima.ducc.common.utils.DuccLoggerComponents;
import org.apache.uima.ducc.common.utils.id.DuccId;
+/**
+ * A class to interpret the $DUCC_HOME/resources/ducc.nodes file
+ *
+ * The file comprises lines of the following types:
+ * 1. #comment
+ * 2. node.domain
+ * 3. import <filename>
+ */
public class DuccNodes {
@@ -170,9 +178,15 @@ public class DuccNodes {
return nodeSet.nodes;
}
- public static void main(String[] args) {
+ public ArrayList<String> get() {
+ String dirResources = IDuccEnv.DUCC_RESOURCES_DIR;
String fileName = IDuccEnv.DUCC_NODES_FILE_NAME;
+ return get(dirResources, fileName);
+ }
+
+ public static void main(String[] args) {
String dirResources = IDuccEnv.DUCC_RESOURCES_DIR;
+ String fileName = IDuccEnv.DUCC_NODES_FILE_NAME;
ArrayList<String> nodes =
DuccNodes.getSysOutInstance().get(dirResources,fileName);
Iterator<String> iterator = nodes.iterator();
while(iterator.hasNext()) {
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/NodeId.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/NodeId.java?rev=1744130&r1=1744129&r2=1744130&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/NodeId.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/NodeId.java
Mon May 16 20:15:37 2016
@@ -18,19 +18,51 @@
*/
package org.apache.uima.ducc.ws.types;
+/**
+ * A class to manage the machine name (ie node identity) of a
+ * resource in the DUCC cluster.
+ */
public class NodeId implements Comparable<NodeId> {
- private String machine;
+ private String machine; // The name of the machine
+ /**
+ * @param machine - the name of the machine, nominally including domain
+ */
public NodeId(String machine) {
this.machine = machine;
}
+ /**
+ * @return the name of the machine w/o the domain
+ */
+ public String getShortName() {
+ String retVal = this.machine;
+ if(retVal != null) {
+ if(retVal.contains(".")) {
+ String expr = "\\.";
+ retVal = retVal.split(expr)[0];
+ }
+ }
+ return retVal;
+ }
+
+ /**
+ * @return the name of the machine including the domain, if present
+ */
+ public String getLongName() {
+ return this.machine;
+ }
+
+ // @return the name of the machine including the domain, if present
+
@Override
public String toString() {
- return this.machine;
+ return getLongName();
}
+ // @return 0 if the long names match
+
@Override
public int compareTo(NodeId nodeId) {
int retVal = 0;
@@ -43,6 +75,8 @@ public class NodeId implements Comparabl
return retVal;
}
+ // @return true if the long names match
+
@Override
public boolean equals(Object object) {
boolean retVal = false;
