Author: degenaro
Date: Mon May 2 18:54:16 2016
New Revision: 1742022
URL: http://svn.apache.org/viewvc?rev=1742022&view=rev
Log:
UIMA-4829 When a DUCC pinger hangs is it reported as Available
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/IServiceMeta.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingDriver.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/Pong.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/IServiceMeta.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/IServiceMeta.java?rev=1742022&r1=1742021&r2=1742022&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/IServiceMeta.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/IServiceMeta.java
Mon May 2 18:54:16 2016
@@ -30,4 +30,5 @@ interface IServiceMeta
public void run();
public void stop();
public ServiceState getServiceState();
+ public long getServiceStatisticsTimestamp();
}
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingDriver.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingDriver.java?rev=1742022&r1=1742021&r2=1742022&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingDriver.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingDriver.java
Mon May 2 18:54:16 2016
@@ -35,6 +35,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Random;
import java.util.Timer;
import java.util.TimerTask;
@@ -43,6 +44,7 @@ import org.apache.uima.ducc.cli.IUiOptio
import org.apache.uima.ducc.common.IServiceStatistics;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.DuccProperties;
+import org.apache.uima.ducc.common.utils.DuccPropertiesResolver;
import org.apache.uima.ducc.common.utils.QuotedOptions;
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.transport.event.common.IDuccState.JobState;
@@ -104,6 +106,7 @@ class PingDriver
int failure_max;
int failure_window;
+ long service_statistics_timestamp = -1;
IServiceStatistics service_statistics = null;
String user;
@@ -292,6 +295,12 @@ class PingDriver
{
// nothing
}
+
+ public long getServiceStatisticsTimestamp()
+ {
+ return service_statistics_timestamp;
+ }
+
public IServiceStatistics getServiceStatistics()
{
@@ -326,6 +335,32 @@ class PingDriver
{
String methodName = "handleStatistics";
+ // ***** ERROR INJECTION *****
+ String key = "ducc.sm.meta.ping.error.injection.missing.percentage";
+ String value =
DuccPropertiesResolver.getInstance().getFileProperty(key);
+ long missingPercentage = 0;
+ try {
+ missingPercentage = Long.parseLong(value);
+ }
+ catch(Exception e) {
+ logger.trace(methodName, sset.getId(), e);
+ }
+ logger.trace(methodName, sset.getId(), key+"="+missingPercentage);
+ if(missingPercentage > 0) {
+ if(missingPercentage < 100) {
+ Random random = new Random();
+ int n = random.nextInt(100);
+ if(n < missingPercentage) {
+ logger.warn(methodName, sset.getId(), "skip pinger
data");
+ return;
+ }
+ logger.warn(methodName, sset.getId(), "keep pinger data");
+ }
+ }
+ // ***** ERROR INJECTION *****
+
+ this.service_statistics_timestamp = response.getTimestamp();
+
this.service_statistics = response.getStatistics();
if ( service_statistics == null ) {
logger.error(methodName, sset.getId(), "Service statics are
null!");
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/Pong.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/Pong.java?rev=1742022&r1=1742021&r2=1742022&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/Pong.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/Pong.java
Mon May 2 18:54:16 2016
@@ -30,6 +30,7 @@ public class Pong
{
private static final long serialVersionUID = 1L;
+ long timestamp = -1;
IServiceStatistics statistics;
int additions;
Long[] deletions;
@@ -39,8 +40,13 @@ public class Pong
public Pong()
{
+ timestamp= System.currentTimeMillis();
}
+ public long getTimestamp() {
+ return timestamp;
+ }
+
public IServiceStatistics getStatistics()
{
return statistics;
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java?rev=1742022&r1=1742021&r2=1742022&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
Mon May 2 18:54:16 2016
@@ -45,6 +45,7 @@ import org.apache.uima.ducc.common.TcpSt
import org.apache.uima.ducc.common.persistence.services.IStateServices;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.DuccProperties;
+import org.apache.uima.ducc.common.utils.SystemPropertyResolver;
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.transport.event.common.DuccWorkJob;
import
org.apache.uima.ducc.transport.event.common.IDuccCompletionType.JobCompletionType;
@@ -128,6 +129,9 @@ public class ServiceSet
// Date of last known succesful ping of the service. 0 means never.
UIMA-4309
long last_ping = 0;
+ boolean notPinging = false;
+ String notPingingReason = null;
+
// Date of last known time any instance made it to Running state. 0 means
never. UIMA-4309
long last_runnable = 0;
@@ -857,7 +861,66 @@ public class ServiceSet
prepareMetaProperties();
stateHandler.updateMetaProperties(id, meta_props);
}
-
+
+ private long pingStabilityDefault = 10;
+ private long pingRateDefault = 60 * 1000;
+
+ private long pingStability = -1;
+ private long pingRate = -1;
+
+ private void configPingStability() {
+ pingStability =
SystemPropertyResolver.getLongProperty("ducc.sm.meta.ping.stability",
pingStabilityDefault);
+ }
+
+ private void configPingRate() {
+ pingRate =
SystemPropertyResolver.getLongProperty("ducc.sm.meta.ping.rate",
pingRateDefault);
+ }
+
+ private void configPing() {
+ configPingRate();
+ configPingStability();
+ }
+
+ /**
+ * If the Service state is Available= but the pinger data has not been
updated
+ * beyond the expiry time then determine the the pinger data is stale
+ *
+ * The expiry time is calculated as pingStability * pingRate, nominally
10 * 60000.
+ */
+ private void determinePingerStatus() {
+ String location = "determinePingerStatus";
+ switch(getState()) {
+ case Available:
+ if(serviceMeta == null) {
+ notPinging = true;
+ notPingingReason = "pinger has not reported";
+ }
+ else {
+ configPing();
+ long pingExpiry = pingStability * pingRate;
+ long now = System.currentTimeMillis();
+ long pingElapsed = now - last_ping;
+ if(pingElapsed > pingExpiry) {
+ notPinging = true;
+ notPingingReason = "pinger data is stale";
+ }
+ else {
+ notPinging = false;
+ notPingingReason = "N/A";
+ }
+ }
+ break;
+ default:
+ notPinging = false;
+ notPingingReason = "N/A";
+ break;
+ }
+ if(notPinging) {
+ logger.info(location, id, notPingingReason);
+ }
+ return;
+ }
+
void prepareMetaProperties()
{
// String methodName = "saveMetaProperties";
@@ -887,13 +950,14 @@ public class ServiceSet
meta_props.setProperty(implementors_key, s);
}
+ determinePingerStatus();
meta_props.put(IStateServices.SvcMetaProps.reference.pname(),
isReferencedStart() ? "true" : "false");
meta_props.put(IStateServices.SvcMetaProps.autostart.pname(),
isAutostart() ? "true" : "false");
meta_props.put(IStateServices.SvcMetaProps.enabled.pname(),
""+enabled);
meta_props.put(IStateServices.SvcMetaProps.service_state.pname(), ""+
getState());
- meta_props.put(IStateServices.SvcMetaProps.ping_active.pname(), "" +
(serviceMeta != null));
+ meta_props.put(IStateServices.SvcMetaProps.ping_active.pname(), "" +
!notPinging);
meta_props.put(IStateServices.SvcMetaProps.service_alive.pname(),
"false");
meta_props.put(IStateServices.SvcMetaProps.service_healthy.pname(),
"false");
@@ -911,7 +975,7 @@ public class ServiceSet
meta_props.put(IStateServices.SvcMetaProps.service_statistics.pname(), "" +
ss.getInfo());
if ( ss.isAlive() ) { // UIMA-4309
- setLastPing(System.currentTimeMillis());
+ setLastPing(serviceMeta.getServiceStatisticsTimestamp());
}
}
}
