Author: degenaro
Date: Tue Apr 19 11:47:29 2016
New Revision: 1739908
URL: http://svn.apache.org/viewvc?rev=1739908&view=rev
Log:
UIMA-4902 DUCC Job Driver (JD) add programmability feature to built-in error
handler
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/logger/
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/logger/LoggerFormatter.java
(with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/logger/ToLog.java
(with props)
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/ErrorHandler.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/ErrorHandler.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/ErrorHandler.java?rev=1739908&r1=1739907&r2=1739908&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/ErrorHandler.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/ErrorHandler.java
Tue Apr 19 11:47:29 2016
@@ -21,14 +21,45 @@ package org.apache.uima.ducc;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.uima.ducc.logger.ToLog;
import org.apache.uima.ducc.user.common.QuotedOptions;
import org.apache.uima.ducc.user.error.iface.Transformer;
import org.apache.uima.ducc.user.jd.JdUser;
public class ErrorHandler implements IErrorHandler {
+ /**
+ * These are the System Properties nominally specified in the
+ * user's job submission (as -D's for driver_jvm_args) which will
+ * be considered for adjusting runtime operational characteristics.
+ */
+ public static enum SystemPropertyNames {
+ JobDriverErrorHandlerMaximumNumberOfTimeoutRetrysPerWorkItem,
+ }
+
+ /**
+ * Return a directive with isKillWorkItem() == false unless and until
+ * the maximumNumberOfTimeoutRetrysPerWorkItem is exceeded.
+ */
+ private AtomicInteger maximumNumberOfTimeoutRetrysPerWorkItem = new
AtomicInteger(0);
+
+ /**
+ * Flag to insure initialization occurs exactly once.
+ */
+ private AtomicBoolean alreadryInitialized = new AtomicBoolean(false);
+
+ /**
+ * A map comprising an entry for each work item with a corresponding
count
+ * of the number of times the work item has been retried
+ */
+ private ConcurrentHashMap<String,AtomicLong> retryMap = new
ConcurrentHashMap<String,AtomicLong>();
+
public enum InitializationDataKey {
KillJobLimit("max_job_errors"),
;
@@ -46,7 +77,6 @@ public class ErrorHandler implements IEr
public String altname() {
return altname;
}
-
};
private static int DefaultJobErrorLimit = JdUser.DefaultJobErrorLimit;
@@ -62,6 +92,56 @@ public class ErrorHandler implements IEr
initialize(initializationData);
}
+ /**
+ * Insure we initialize exactly once
+ */
+ private void initializeOnce() {
+ synchronized(ErrorHandler.class) {
+ if(!alreadryInitialized.get()) {
+ Properties systemProperties =
System.getProperties();
+ initTimeoutRetrys(systemProperties);
+ alreadryInitialized.set(true);
+ }
+ }
+ }
+
+ /**
+ * Use the user specified
-DJobDriverErrorHandlerMaximumNumberOfTimeoutRetrysPerWorkItem to set
+ * max timeouts per work item, if specified otherwise 0 meaning no
retrys
+ */
+ private void initTimeoutRetrys(Properties systemProperties) {
+ String key =
SystemPropertyNames.JobDriverErrorHandlerMaximumNumberOfTimeoutRetrysPerWorkItem.name();
+ if(systemProperties != null) {
+ if(systemProperties.containsKey(key)) {
+ String value =
systemProperties.getProperty(key);
+ try {
+ int integer = Integer.parseInt(value);
+ if(integer < 0) {
+ String text = "Invalid:
"+key+"="+value;
+
ToLog.info(ErrorHandler.class,text);
+ }
+ else {
+
maximumNumberOfTimeoutRetrysPerWorkItem.set(integer);;
+ String text = "Override:
"+key+"="+maximumNumberOfTimeoutRetrysPerWorkItem.get();
+
ToLog.info(ErrorHandler.class,text);
+ }
+ }
+ catch(Exception e) {
+ String text = "Invalid: "+key+"="+value;
+ ToLog.info(ErrorHandler.class,text);
+ }
+ }
+ else {
+ String text = "Default:
"+key+"="+maximumNumberOfTimeoutRetrysPerWorkItem.get();
+ ToLog.info(ErrorHandler.class,text);
+ }
+ }
+ else {
+ String text = "Default:
"+key+"="+maximumNumberOfTimeoutRetrysPerWorkItem.get();
+ ToLog.info(ErrorHandler.class,text);
+ }
+ }
+
private Map<String, String> parse(String initializationData) {
Map<String, String> map = new HashMap<String, String>();
try {
@@ -116,6 +196,7 @@ public class ErrorHandler implements IEr
@Override
public IErrorHandlerDirective handle(String serializedCAS, Object
object) {
+ initializeOnce();
ErrorHandlerDirective jdUserDirective = new
ErrorHandlerDirective();
try {
Throwable userThrowable = null;
@@ -127,15 +208,36 @@ public class ErrorHandler implements IEr
if(object instanceof Exception) {
userThrowable = (Throwable) object;
userThrowable.getClass();
+ ToLog.info(ErrorHandler.class,
serializedCAS);
+ ToLog.info(ErrorHandler.class,
userThrowable);
+ if(serializedCAS != null) {
+
retryMap.putIfAbsent(serializedCAS, new AtomicLong(0));
+ AtomicLong retryCount =
retryMap.get(serializedCAS);
+ long count =
retryCount.incrementAndGet();
+ if(count <=
maximumNumberOfTimeoutRetrysPerWorkItem.get()) {
+
jdUserDirective.resetKillWorkItem();
+ String text = "retry #
"+count+" of "+maximumNumberOfTimeoutRetrysPerWorkItem.get()+" for:
"+serializedCAS;
+
ToLog.info(ErrorHandler.class,text);
+ }
+ else {
+
jobErrorCount.incrementAndGet();
+ }
+ }
+ else {
+ jobErrorCount.incrementAndGet();
+ }
}
// User code exception
else {
Object byteArray = object;
userThrowable =
Transformer.deserialize(byteArray);
userThrowable.getClass();
+ jobErrorCount.incrementAndGet();
}
}
- jobErrorCount.incrementAndGet();
+ else {
+ jobErrorCount.incrementAndGet();
+ }
if(jobErrorCount.get() > jobErrorLimit.get()) {
jdUserDirective.setKillJob();
}
@@ -143,6 +245,16 @@ public class ErrorHandler implements IEr
catch(Exception e) {
e.printStackTrace();
}
+ StringBuffer sb = new StringBuffer();
+ sb.append("KillJob: ");
+ sb.append(jdUserDirective.isKillJob());
+ sb.append(" ");
+ sb.append("KillProcess: ");
+ sb.append(jdUserDirective.isKillProcess());
+ sb.append(" ");
+ sb.append("KillKillWorkItem: ");
+ sb.append(jdUserDirective.isKillWorkItem());
+ ToLog.info(ErrorHandler.class, sb.toString());
return jdUserDirective;
}
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/logger/LoggerFormatter.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/logger/LoggerFormatter.java?rev=1739908&view=auto
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/logger/LoggerFormatter.java
(added)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/logger/LoggerFormatter.java
Tue Apr 19 11:47:29 2016
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.logger;
+
+import java.util.logging.LogRecord;
+import java.util.logging.SimpleFormatter;
+
+/**
+ * This class is used to format messages written to ErrorHandler.log
+ * in the user's Job log directory.
+ */
+
+public class LoggerFormatter extends SimpleFormatter {
+ public String format(LogRecord record) {
+ return new java.util.Date() + " " + record.getLevel() + " " +
record.getMessage() + "\r\n";
+ }
+}
Propchange:
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/logger/LoggerFormatter.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/logger/LoggerFormatter.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/logger/ToLog.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/logger/ToLog.java?rev=1739908&view=auto
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/logger/ToLog.java
(added)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/logger/ToLog.java
Tue Apr 19 11:47:29 2016
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.logger;
+
+import java.io.File;
+import java.util.logging.FileHandler;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * This class is used to record messages into file ErrorHandler.log
+ * in the user's Job log directory.
+ */
+
+public class ToLog {
+
+ private static String getFilePath(Class<?> clazz) {
+ String filePath = null;
+ String key = "ducc.process.log.dir";
+ String value = System.getProperty(key);
+ if(value != null) {
+ filePath = value;
+ if(!filePath.endsWith(File.separator)) {
+ filePath = filePath+File.separator;
+ }
+ filePath = filePath+clazz.getSimpleName()+".log";
+ }
+ return filePath;
+ }
+
+ private static Logger create(Class<?> clazz, String filePath) {
+ Logger logger = Logger.getLogger(clazz.getCanonicalName());
+ try {
+ boolean append = true;
+ FileHandler fileHandler = new FileHandler(filePath,
append);
+ LoggerFormatter loggerFormatter = new LoggerFormatter();
+ fileHandler.setFormatter(loggerFormatter);
+ logger.addHandler(fileHandler);
+ logger.setUseParentHandlers(false);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ }
+ return logger;
+ }
+
+ private static Logger getLogger(Class<?> clazz, String filePath) {
+ Logger logger = null;
+ if(filePath != null) {
+ logger = create(clazz, filePath);
+ }
+ return logger;
+ }
+
+ private static Logger getLogger(Class<?> clazz) {
+ String filePath = getFilePath(clazz);
+ Logger logger = getLogger(clazz, filePath);
+ return logger;
+ }
+
+ private static void close(Logger logger) {
+ Handler[] handlers = logger.getHandlers();
+ if(handlers != null) {
+ for(Handler handler : handlers) {
+ handler.close();
+ }
+ }
+ }
+
+ /**
+ * Write a String message into ErrorHandler.log file
+ */
+ public static void info(Class<?> clazz, String text) {
+ if(clazz != null) {
+ if(text != null) {
+ Logger logger = getLogger(clazz);
+ if(logger != null) {
+ logger.info(text);
+ close(logger);
+ }
+ }
+ }
+ }
+
+ /**
+ * Write a Throwable message into ErrorHandler.log file
+ */
+ public static void info(Class<?> clazz, Throwable t) {
+ if(clazz != null) {
+ if(t != null) {
+ Logger logger = getLogger(clazz);
+ if(logger != null) {
+ logger.log(Level.INFO, t.getMessage(),
t);
+ close(logger);
+ }
+ }
+ }
+ }
+
+}
Propchange:
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/logger/ToLog.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
uima/sandbox/uima-ducc/trunk/uima-ducc-user/src/main/java/org/apache/uima/ducc/logger/ToLog.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
