Author: cwiklik
Date: Mon Jul 30 14:43:05 2018
New Revision: 1837063
URL: http://svn.apache.org/viewvc?rev=1837063&view=rev
Log:
UIMA-5843 added error window support
Modified:
uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java
uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/container/FlagsHelper.java
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/ServiceConfiguration.java
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/AbstractServiceProcessor.java
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaAsServiceProcessor.java
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaProcessResult.java
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaServiceProcessor.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/cli/JobSpecificationProperties.java
Modified:
uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java
Mon Jul 30 14:43:05 2018
@@ -80,6 +80,8 @@ public class DuccJobSubmit
UiOption.ProcessInitializationFailuresCap,
UiOption.ProcessFailuresLimit,
UiOption.ProcessPipelineCount,
+ UiOption.ProcessErrorWindowThreshold,
+ UiOption.ProcessErrorWindowSize,
UiOption.ProcessPerItemTimeMax,
UiOption.ProcessInitializationTimeMax,
@@ -163,6 +165,7 @@ public class DuccJobSubmit
{
init (this.getClass().getName(), opts, args, jobRequestProperties,
consoleCb);
check_descriptor_options();
+ check_error_window_definition();
if (isAllInOne()) {
allInOneLauncher = new AllInOneLauncher(userSpecifiedProperties,
consoleCb); // Pass the already fixed-up user properties
}
@@ -182,12 +185,43 @@ public class DuccJobSubmit
throws Exception
{
init (this.getClass().getName(), opts, props, jobRequestProperties,
consoleCb);
+
check_descriptor_options();
+ check_error_window_definition();
if (isAllInOne()) {
allInOneLauncher = new AllInOneLauncher(userSpecifiedProperties,
consoleCb); // Pass the already fixed-up user properties
}
}
-
+ private void check_error_window_definition() throws
IllegalArgumentException {
+ int windowSize=0;
+ int maxThreshold=0;
+
+ boolean errorWindowSizeDefined =
jobRequestProperties.containsKey(UiOption.ProcessErrorWindowSize.pname());
+ boolean errorWindowThresholdDefined =
jobRequestProperties.containsKey(UiOption.ProcessErrorWindowThreshold.pname());
+
+ if ( errorWindowSizeDefined ) {
+ try {
+ windowSize =
jobRequestProperties.getIntProperty(UiOption.ProcessErrorWindowSize.pname());
+ } catch( NumberFormatException e) {
+ throw new
IllegalArgumentException("process_error_window_size must be a postive number -
you've provided:"+
+
jobRequestProperties.getIntProperty(UiOption.ProcessErrorWindowSize.pname()));
+ }
+ }
+ if ( errorWindowThresholdDefined ) {
+ try {
+ maxThreshold =
jobRequestProperties.getIntProperty(UiOption.ProcessErrorWindowThreshold.pname());
+ } catch( NumberFormatException e) {
+ throw new
IllegalArgumentException("process_error_window_threshold must be a postive
number - you've provided:"+
+
jobRequestProperties.getIntProperty(UiOption.ProcessErrorWindowThreshold.pname()));
+ }
+ }
+ if ( maxThreshold > windowSize ) {
+ throw new
IllegalArgumentException("process_error_window_threshold must be smaller/equal
than/to process_error_window_size - you've provided:"+
+ "
process_error_window_threshold="+jobRequestProperties.getIntProperty(UiOption.ProcessErrorWindowThreshold.pname())
+
+ "
process_error_window_size="+jobRequestProperties.getIntProperty(UiOption.ProcessErrorWindowSize.pname()));
+
+ }
+ }
/*
* If preemptable change to a non-preemptable scheduling class.
* If none provided use the default fixed class
Modified:
uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java
Mon Jul 30 14:43:05 2018
@@ -517,7 +517,20 @@ public interface IUiOptions
public String description() { return "Number of pipelines per
process (i.e. simultaneously processed work-items)."; }
public String argname() { return "integer"; }
public String deflt() { return "4"; }
- },
+ },
+ ProcessErrorWindowThreshold {
+ public String pname() { return
JobSpecificationProperties.key_process_error_threshold; }
+ public String description() { return "Maximum number of process
errors per process."; }
+ public String argname() { return "integer"; }
+ public String deflt() { return "1"; }
+ },
+
+ ProcessErrorWindowSize {
+ public String pname() { return
JobSpecificationProperties.key_process_error_window; }
+ public String description() { return "Size of error window used in
conjuntion with maximum number of process errors."; }
+ public String argname() { return "integer"; }
+ public String deflt() { return "1"; }
+ },
ProcessPerItemTimeMax {
public String pname() { return
JobSpecificationProperties.key_process_per_item_time_max; }
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/container/FlagsHelper.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/container/FlagsHelper.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/container/FlagsHelper.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/container/FlagsHelper.java
Mon Jul 30 14:43:05 2018
@@ -48,6 +48,8 @@ public class FlagsHelper {
JpCmOverrides,
JpDd,
JpThreadCount,
+ JpErrorThreshold,
+ JpErrorWindowSize,
JpType, // { uima,
uima-as }
JpProcessorClass("processor.class"),
UserClasspath, //
path1.class:path2.jar:path3/*:...
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
Mon Jul 30 14:43:05 2018
@@ -641,6 +641,15 @@ public class JobFactory implements IJobF
// </flags for JP to build Aggregate>
}
+ String process_error_threshold =
jobRequestProperties.getProperty(JobSpecificationProperties.key_process_error_threshold);
+ if(process_error_threshold != null) {
+ addDashD(pipelineCommandLine,
FlagsHelper.Name.JpErrorThreshold, process_error_threshold);
+ }
+ String process_error_window =
jobRequestProperties.getProperty(JobSpecificationProperties.key_process_error_window);
+ if(process_error_window != null) {
+ addDashD(pipelineCommandLine,
FlagsHelper.Name.JpErrorWindowSize, process_error_window);
+ }
+
String process_thread_count =
jobRequestProperties.getProperty(JobSpecificationProperties.key_process_pipeline_count);
if(process_thread_count != null) {
addDashD(pipelineCommandLine,
FlagsHelper.Name.JpThreadCount, process_thread_count);
Modified:
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/ServiceConfiguration.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/ServiceConfiguration.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/ServiceConfiguration.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/ServiceConfiguration.java
Mon Jul 30 14:43:05 2018
@@ -20,6 +20,7 @@
package org.apache.uima.ducc.ps.service;
import org.apache.uima.ducc.ps.service.errors.ServiceInitializationException;
+import org.apache.uima.ducc.ps.service.jmx.JMXAgent;
public class ServiceConfiguration {
private String clientURL;
@@ -47,7 +48,14 @@ public class ServiceConfiguration {
private String maxErrors;
private String errorWindowSize;
private ClassLoader sysCL=null;
-
+ private String processType;
+
+ public String getProcessType() {
+ return processType;
+ }
+ public void setProcessType(String processType) {
+ this.processType = processType;
+ }
public String getMaxErrors() {
return maxErrors;
}
@@ -240,7 +248,7 @@ public class ServiceConfiguration {
serviceJmxConnectURL = System.getProperty("ducc.jmx.port");
customRegistryClass =
System.getProperty("ducc.deploy.registry.class");
customProcessorClass =
System.getProperty("ducc.deploy.custom.processor.class");
-
+ processType =
System.getProperty("ducc.deploy.components");//=job-process
duccHome = System.getenv("DUCC_HOME");
jobId = System.getenv("DUCC_JOBID");
duccProcessId = System.getenv("DUCC_PROCESSID");
Modified:
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/AbstractServiceProcessor.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/AbstractServiceProcessor.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/AbstractServiceProcessor.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/AbstractServiceProcessor.java
Mon Jul 30 14:43:05 2018
@@ -37,8 +37,10 @@ public class AbstractServiceProcessor {
protected int maxErrors=1; // default is to fail on 1st error
protected int windowSize=1;
protected int DEFAULT_INIT_DELAY=30000;
-
- protected IServiceErrorHandler getErrorHandler() {
+
+
+ protected IServiceErrorHandler getErrorHandler(Logger logger) {
+ logger.log(Level.INFO, "Error Handler Configuration:
threshold="+maxErrors+" windowSize="+windowSize);
// concrete implementation of this abstract class should
// provide a way to set maxErrors and windowSize. The
IServiceProcessor
// provides a setter method for overriding default values
@@ -66,4 +68,7 @@ public class AbstractServiceProcessor {
return serializerMap.get(Thread.currentThread().getId());
}
+ protected void stop() throws Exception {
+
+ }
}
Modified:
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaAsServiceProcessor.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaAsServiceProcessor.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaAsServiceProcessor.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaAsServiceProcessor.java
Mon Jul 30 14:43:05 2018
@@ -107,6 +107,7 @@ public class UimaAsServiceProcessor exte
public UimaAsServiceProcessor(String[] args, ServiceConfiguration
serviceConfiguration) {
this.args = args;
this.serviceConfiguration = serviceConfiguration;
+
// start a thread which will collect AE initialization state
launchStateInitializationCollector();
}
@@ -162,7 +163,7 @@ public class UimaAsServiceProcessor exte
}
// Needed to resolve ${queue.name} placeholder in DD
generated by DUCC
System.setProperty(queuePropertyName, endpointName);
- errorHandler = getErrorHandler();
+ errorHandler = getErrorHandler(logger);
// generate Spring context file once
synchronized (UimaAsServiceProcessor.class) {
@@ -462,6 +463,7 @@ public class UimaAsServiceProcessor exte
Method waitMethod =
classToLaunch.getMethod("waitUntilStopped");
waitMethod.invoke(brokerInstance);
brokerRunning = false;
+ super.stop();
} catch (Exception e) {
logger.log(Level.WARNING, "stop", e);
Modified:
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaProcessResult.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaProcessResult.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaProcessResult.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaProcessResult.java
Mon Jul 30 14:43:05 2018
@@ -55,5 +55,9 @@ public class UimaProcessResult implement
exception.printStackTrace(new PrintWriter(sw));
return sw.toString();
}
+ @Override
+ public Exception getExceptionObject() {
+ return exception;
+ }
}
Modified:
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaServiceProcessor.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaServiceProcessor.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaServiceProcessor.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaServiceProcessor.java
Mon Jul 30 14:43:05 2018
@@ -143,7 +143,7 @@ public class UimaServiceProcessor extend
logger.log(Level.FINE, "Process Thread:"+
Thread.currentThread().getName()+" Initializing AE");
}
- errorHandler = getErrorHandler();
+ errorHandler = getErrorHandler(logger);
try {
// multiple threads may call this method. Send
initializing state once
@@ -282,6 +282,8 @@ public class UimaServiceProcessor extend
if ( ae != null ) {
ae.destroy();
}
+ super.stop();
+
} catch( Exception e) {
logger.log(Level.WARNING, "stop", e);
}
Modified:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/cli/JobSpecificationProperties.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/cli/JobSpecificationProperties.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/cli/JobSpecificationProperties.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/cli/JobSpecificationProperties.java
Mon Jul 30 14:43:05 2018
@@ -65,6 +65,8 @@ public class JobSpecificationProperties
public static String key_process_initialization_failures_cap =
"process_initialization_failures_cap";
public static String key_process_failures_limit =
"process_failures_limit";
public static String key_process_pipeline_count =
"process_pipeline_count";
+ public static String key_process_error_threshold =
"process_error_window_threshold";
+ public static String key_process_error_window =
"process_error_window_size";
/*
public static String key_process_get_meta_time_max =
"process_get_meta_time_max";
