Author: degenaro
Date: Thu Jun 30 12:13:01 2016
New Revision: 1750783
URL: http://svn.apache.org/viewvc?rev=1750783&view=rev
Log:
UIMA-4616 The uima-ducc-user jar should be added to the head of the user's
classpath
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccUser.java
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccUser.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccUser.java?rev=1750783&r1=1750782&r2=1750783&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccUser.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccUser.java
Thu Jun 30 12:13:01 2016
@@ -31,6 +31,9 @@ public interface IDuccUser {
//
DUCC_IP("DUCC_IP"),
DUCC_NODENAME("DUCC_NODENAME"),
+ //
+ DUCC_USER_CP_PREPEND("DUCC_USER_CP_PREPEND"),
+ //
USER("USER"),
;
Modified:
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex?rev=1750783&r1=1750782&r2=1750783&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex
(original)
+++
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part2/cli/ducc-submit.tex
Thu Jun 30 12:13:01 2016
@@ -127,6 +127,18 @@
\\Note: When used as a CLI option, the environment string must
usually be
quoted to protect it from the shell.
+
+ The following cause special runtime behavior.
+ They are considered experimental and are not guaranteed
+ to be effective from release to release.
+
+ \begin{enumerate}
+ \item[DUCC\_USER\_CP\_PREPEND {[path-to-ducc-jars-and-classes]} ]
+ If specified, this path is used to supply the DUCC classes required for
running
+ the Job Driver and Job Process(es), normally set to
$DUCC_HOME/lib/uima-ducc/users/*.
+ \end{enumerate}
+
+
\item[$--$help ]
Prints the usage text to the console.
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java?rev=1750783&r1=1750782&r2=1750783&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
Thu Jun 30 12:13:01 2016
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.uima.ducc.common.IDuccEnv;
+import org.apache.uima.ducc.common.IDuccUser;
import org.apache.uima.ducc.common.NodeIdentity;
import org.apache.uima.ducc.common.config.CommonConfiguration;
import org.apache.uima.ducc.common.container.FlagsHelper;
@@ -219,10 +220,21 @@ public class JobFactory implements IJobF
}
}
- private String addUimaDucc(String cp) {
+ private String getPrependUserCP(DuccId jobid, JobRequestProperties
jobSpec) {
+ String envKey =
IDuccUser.EnvironmentVariable.DUCC_USER_CP_PREPEND.value();
+ String prependDefault =
IDuccEnv.DUCC_HOME+File.separator+"lib"+File.separator+"uima-ducc"+File.separator+"user"+File.separator+"*";
+ String retVal = JobFactoryHelper.getEnvVal(jobid, jobSpec,
envKey, prependDefault);
+ return retVal;
+ }
+
+ /**
+ * @param prependCP - DUCC classes needed to run JD/JP
+ * @param cp - user classes needed to run JD/JP
+ * @return the entirety of classes needed to run JD/JP: DUCC first,
user second
+ */
+ private String addUimaDucc(String prependCP, String cp) {
StringBuffer sb = new StringBuffer();
- String prepend =
IDuccEnv.DUCC_HOME+File.separator+"lib"+File.separator+"uima-ducc"+File.separator+"user"+File.separator+"*";
- sb.append(prepend);
+ sb.append(prependCP);
sb.append(File.pathSeparator);
if(cp != null) {
String tcp = cp.trim();
@@ -277,8 +289,9 @@ public class JobFactory implements IJobF
jcl.addOption(opt);
}
// add userCP
+ String prependUserCP = getPrependUserCP(jobid,
jobRequestProperties);
String userCP =
jobRequestProperties.getProperty(JobSpecificationProperties.key_classpath);
- userCP = addUimaDucc(userCP);
+ userCP = addUimaDucc(prependUserCP, userCP);
opt = FlagsHelper.Name.UserClasspath.dname()+"="+userCP;
jcl.addOption(opt);
// add WorkItemTimeout
@@ -445,6 +458,7 @@ public class JobFactory implements IJobF
private DuccWorkJob create(CommonConfiguration common,
JobRequestProperties jobRequestProperties, DuccWorkJob job) {
String methodName = "create";
jobRequestProperties.normalize();
+ DuccId jobid = job.getDuccId();
DuccType duccType = job.getDuccType();
// Service Deployment Type
if(jobRequestProperties.containsKey(ServiceRequestProperties.key_service_type_custom))
{
@@ -580,8 +594,9 @@ public class JobFactory implements IJobF
dump(job, uimaAggregate);
}
// user CP
+ String prependUserCP = getPrependUserCP(jobid,
jobRequestProperties);
String userCP =
jobRequestProperties.getProperty(JobSpecificationProperties.key_classpath);
- userCP = addUimaDucc(userCP);
+ userCP = addUimaDucc(prependUserCP, userCP);
pipelineCommandLine.setClasspath(userCP);
// jvm args
String process_jvm_args =
jobRequestProperties.getProperty(JobSpecificationProperties.key_process_jvm_args);
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java?rev=1750783&r1=1750782&r2=1750783&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java
Thu Jun 30 12:13:01 2016
@@ -18,12 +18,18 @@
*/
package org.apache.uima.ducc.orchestrator.factory;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.DuccLoggerComponents;
import org.apache.uima.ducc.common.utils.DuccPropertiesResolver;
+import org.apache.uima.ducc.common.utils.QuotedOptions;
import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.transport.event.cli.JobRequestProperties;
+import org.apache.uima.ducc.transport.event.cli.JobSpecificationProperties;
public class JobFactoryHelper {
@@ -95,4 +101,73 @@ public class JobFactoryHelper {
logger.debug(location, jobid, retVal);
return retVal;
}
+
+ /**
+ * @param jobid the DuccId for the Job
+ * @param jobSpec the Job specification
+ * @return environment variable map derived from user specified
environment variable string
+ */
+ public static Map<String, String> getEnvMap(DuccId jobid,
JobRequestProperties jobSpec) {
+ String location = "getEnvMap";
+ Map<String, String> retVal = new HashMap<String,String>();
+ if(jobSpec != null) {
+ String environmentVariables =
jobSpec.getProperty(JobSpecificationProperties.key_environment);
+ if(environmentVariables != null) {
+ ArrayList<String> envVarList =
QuotedOptions.tokenizeList(environmentVariables, true);
+ try {
+ Map<String, String> envMap =
QuotedOptions.parseAssignments(envVarList, 0);
+ if(envMap != null) {
+ retVal = envMap;
+ }
+ }
+ catch (IllegalArgumentException e) {
+ logger.warn(location, jobid,"Invalid environment syntax
in: " + environmentVariables);
+ }
+ }
+ else {
+ logger.trace(location, jobid,
"environmentVariables="+environmentVariables);
+ }
+ }
+ else {
+ logger.debug(location, jobid, "jobSpec="+jobSpec);
+ }
+ return retVal;
+ }
+
+ /**
+ * @param jobid the DuccId for the Job
+ * @param jobSpec the Job specification
+ * @param envKey the environment variable key (name)
+ * @return the environment variable value for the specified key (name)
+ */
+ public static String getEnvVal(DuccId jobid, JobRequestProperties
jobSpec, String envKey) {
+ String location = "getEnvVal";
+ String retVal = null;
+ if(envKey != null) {
+ Map<String, String> envMap = getEnvMap(jobid, jobSpec);
+ retVal = envMap.get(envKey);
+ logger.debug(location, jobid, "envKey="+envKey+"
"+"envVal="+retVal);
+ }
+ else {
+ logger.debug(location, jobid, "envKey="+envKey);
+ }
+ return retVal;
+ }
+
+ /**
+ * @param jobid the DuccId for the Job
+ * @param jobSpec the Job specification
+ * @param envKey the environment variable key (name)
+ * @param envVal the environment variable default value
+ * @return the environment variable value for the specified key (name)
+ */
+ public static String getEnvVal(DuccId jobid, JobRequestProperties
jobSpec, String envKey, String envVal) {
+ String location = "getEnvVal";
+ String retVal = getEnvVal(jobid, jobSpec, envKey);
+ if(retVal == null) {
+ retVal = envVal;
+ logger.debug(location, jobid, "envKey="+envKey+"
"+"envVal="+retVal+" "+"(default)");
+ }
+ return retVal;
+ }
}
