Author: tucu
Date: Thu May 10 04:08:39 2012
New Revision: 1336477
URL: http://svn.apache.org/viewvc?rev=1336477&view=rev
Log:
OOZIE-830 add support for multiple/configurable sharelibs for each action type
(tucu)
Modified:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
incubator/oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
incubator/oozie/trunk/release-log.txt
Modified:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java?rev=1336477&r1=1336476&r2=1336477&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
(original)
+++
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
Thu May 10 04:08:39 2012
@@ -84,4 +84,16 @@ public class DistcpActionExecutor extend
}
return str;
}
+
+ /**
+ * Return the sharelib name for the action.
+ *
+ * @return returns <code>distcp</code>.
+ * @param actionXml
+ */
+ @Override
+ protected String getDefaultShareLibName(Element actionXml) {
+ return "distcp";
+ }
+
}
Modified:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java?rev=1336477&r1=1336476&r2=1336477&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
(original)
+++
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
Thu May 10 04:08:39 2012
@@ -123,13 +123,13 @@ public class HiveActionExecutor extends
}
/**
- * Return the sharelib postfix for the action.
+ * Return the sharelib name for the action.
*
- * @param context executor context.
- * @param actionXml the action XML.
- * @return the action sharelib post fix, this implementation returns
<code>hive</code>.
+ * @return returns <code>hive</code>.
+ * @param actionXml
*/
- protected String getShareLibPostFix(Context context, Element actionXml) {
+ @Override
+ protected String getDefaultShareLibName(Element actionXml) {
return "hive";
}
Modified:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java?rev=1336477&r1=1336476&r2=1336477&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
(original)
+++
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
Thu May 10 04:08:39 2012
@@ -376,13 +376,13 @@ public class JavaActionExecutor extends
}
}
- protected void addShareLib(Path appPath, Configuration conf, String
actionShareLibPostfix)
+ protected void addShareLib(Path appPath, Configuration conf, String
actionShareLibName)
throws ActionExecutorException {
- if (actionShareLibPostfix != null) {
+ if (actionShareLibName != null) {
try {
Path systemLibPath =
Services.get().get(WorkflowAppService.class).getSystemLibPath();
if (systemLibPath != null) {
- Path actionLibPath = new Path(systemLibPath,
actionShareLibPostfix);
+ Path actionLibPath = new Path(systemLibPath,
actionShareLibName);
String user = conf.get("user.name");
String group = conf.get("group.name");
FileSystem fs =
@@ -460,7 +460,7 @@ public class JavaActionExecutor extends
// Action sharelibs are only added if user has specified to use system
libpath
if (wfJobConf.getBoolean(OozieClient.USE_SYSTEM_LIBPATH, false)) {
// add action specific sharelibs
- addShareLib(appPath, conf, getShareLibPostFix(context, actionXml));
+ addShareLib(appPath, conf, getShareLibName(context, actionXml,
conf));
}
}
@@ -497,6 +497,10 @@ public class JavaActionExecutor extends
JobConf launcherJobConf = createBaseHadoopConf(context, actionXml);
setupLauncherConf(launcherJobConf, actionXml, appPathRoot,
context);
+ String actionShareLibProperty = actionConf.get(ACTION_SHARELIB_FOR
+ getType());
+ if (actionShareLibProperty != null) {
+ launcherJobConf.set(ACTION_SHARELIB_FOR + getType(),
actionShareLibProperty);
+ }
setLibFilesArchives(context, actionXml, appPathRoot,
launcherJobConf);
String jobName =
XLog.format("oozie:launcher:T={0}:W={1}:A={2}:ID={3}", getType(),
context.getWorkflow()
.getAppName(), action.getName(),
context.getWorkflow().getId());
@@ -1076,22 +1080,60 @@ public class JavaActionExecutor extends
return FINAL_STATUS.contains(externalStatus);
}
+
/**
- * Return the sharelib postfix for the action.
+ * Return the sharelib name for the action.
* <p/>
* If <code>NULL</code> or emtpy, it means that the action does not use
the action
* sharelib.
* <p/>
* If a non-empty string, i.e. <code>foo</code>, it means the action uses
the
- * action sharelib subdirectory <code>foo</code> and all JARs in the
<code>foo</code>
- * directory will be in the action classpath.
+ * action sharelib subdirectory <code>foo</code> and all JARs in the
sharelib
+ * <code>foo</code> directory will be in the action classpath.
+ * <p/>
+ * The resolution is done using the following precedence order:
+ * <ul>
+ * <li><b>action.sharelib.for.#ACTIONTYPE#</b> in the action
configuration</li>
+ * <li><b>action.sharelib.for.#ACTIONTYPE#</b> in the job
configuration</li>
+ * <li><b>action.sharelib.for.#ACTIONTYPE#</b> in the oozie
configuration</li>
+ * <li>Action Executor <code>getDefaultShareLibName()</code>
method</li>
+ * </ul>
+ *
*
* @param context executor context.
- * @param actionXml the action XML.
- * @return the action sharelib post fix, this implementation returns
<code>NULL</code>.
+ * @param actionXml
+ *@param conf action configuration. @return the action sharelib name.
*/
- protected String getShareLibPostFix(Context context, Element actionXml) {
- return null;
+ protected String getShareLibName(Context context, Element actionXml,
Configuration conf) {
+ String name = conf.get(ACTION_SHARELIB_FOR + getType());
+ if (name == null) {
+ try {
+ XConfiguration jobConf = new XConfiguration(new
StringReader(context.getWorkflow().getConf()));
+ name = jobConf.get(ACTION_SHARELIB_FOR + getType());
+ if (name == null) {
+ name = Services.get().getConf().get(ACTION_SHARELIB_FOR +
getType());
+ if (name == null) {
+ name = getDefaultShareLibName(actionXml);
+ }
+ }
+ }
+ catch (IOException ex) {
+ throw new RuntimeException("It cannot happen, " +
ex.toString(), ex);
+ }
+ }
+ return name;
}
+ private final static String ACTION_SHARELIB_FOR =
"oozie.action.sharelib.for.";
+
+
+ /**
+ * Returns the default sharelib name for the action if any.
+ *
+ * @param actionXml the action XML fragment.
+ * @return the sharelib name for the action, <code>NULL</code> if none.
+ */
+ protected String getDefaultShareLibName(Element actionXml) {
+ return null;
+ }
}
Modified:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java?rev=1336477&r1=1336476&r2=1336477&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
(original)
+++
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
Thu May 10 04:08:39 2012
@@ -20,7 +20,6 @@ package org.apache.oozie.action.hadoop;
import java.io.IOException;
import java.io.StringReader;
import java.util.List;
-import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -37,7 +36,6 @@ import org.apache.oozie.util.XmlUtils;
import org.jdom.Element;
import org.jdom.Namespace;
import org.json.simple.JSONObject;
-import org.mortbay.util.ajax.JSON;
public class MapReduceActionExecutor extends JavaActionExecutor {
@@ -227,14 +225,13 @@ public class MapReduceActionExecutor ext
}
/**
- * Return the sharelib postfix for the action.
+ * Return the sharelib name for the action.
*
- * @param context executor context.
- * @param actionXml the action XML.
- * @return the action sharelib post fix, this implementation returns
<code>NULL</code>
- * or <code>streaming</code> if the mapreduce action is streaming.
+ * @return returns <code>streaming</code> if mapreduce-streaming action,
<code>NULL</code> otherwise.
+ * @param actionXml
*/
- protected String getShareLibPostFix(Context context, Element actionXml) {
+ @Override
+ protected String getDefaultShareLibName(Element actionXml) {
Namespace ns = actionXml.getNamespace();
return (actionXml.getChild("streaming", ns) != null) ?
"mapreduce-streaming" : null;
}
Modified:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java?rev=1336477&r1=1336476&r2=1336477&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
(original)
+++
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
Thu May 10 04:08:39 2012
@@ -26,14 +26,11 @@ import org.apache.oozie.action.ActionExe
import org.apache.oozie.client.XOozieClient;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.service.HadoopAccessorException;
-import org.apache.oozie.util.ClassUtils;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.XLog;
-import org.apache.oozie.util.XmlUtils;
import org.jdom.Element;
import org.jdom.Namespace;
import org.jdom.JDOMException;
-import org.mortbay.log.Log;
import java.io.BufferedReader;
import java.io.IOException;
@@ -203,12 +200,11 @@ public class PigActionExecutor extends J
/**
* Return the sharelib postfix for the action.
*
- * @param context executor context.
- * @param actionXml the action XML.
- * @return the action sharelib post fix, this implementation returns
<code>pig</code>.
+ * @return returns <code>pig</code>.
+ * @param actionXml
*/
@Override
- protected String getShareLibPostFix(Context context, Element actionXml) {
+ protected String getDefaultShareLibName(Element actionXml) {
return "pig";
}
Modified:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java?rev=1336477&r1=1336476&r2=1336477&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
(original)
+++
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
Thu May 10 04:08:39 2012
@@ -114,13 +114,13 @@ public class SqoopActionExecutor extends
/**
- * Return the sharelib postfix for the action.
+ * Return the sharelib name for the action.
*
- * @param context executor context.
- * @param actionXml the action XML.
- * @return the action sharelib post fix, this implementation returns
<code>hive</code>.
+ * @return returns <code>sqoop</code>.
+ * @param actionXml
*/
- protected String getShareLibPostFix(Context context, Element actionXml) {
+ @Override
+ protected String getDefaultShareLibName(Element actionXml) {
return "sqoop";
}
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java?rev=1336477&r1=1336476&r2=1336477&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
Thu May 10 04:08:39 2012
@@ -255,6 +255,7 @@ public class TestJavaActionExecutor exte
actionXml = XmlUtils.parseXml("<java>" + "<job-tracker>" +
getJobTrackerUri() + "</job-tracker>" +
"<name-node>" + getNameNodeUri() + "</name-node>
<configuration>" +
"<property><name>mapred.job.queue.name</name><value>AQ</value></property>" +
+
"<property><name>oozie.action.sharelib.for.java</name><value>sharelib-java</value></property>"
+
"</configuration>" + "<main-class>MAIN-CLASS</main-class>" +
"</java>");
actionConf = ae.createBaseHadoopConf(context, actionXml);
@@ -262,6 +263,7 @@ public class TestJavaActionExecutor exte
conf = ae.createLauncherConf(getFileSystem(), context, action,
actionXml, actionConf);
assertEquals("AQ", conf.get("mapred.job.queue.name"));
assertEquals("AQ", actionConf.get("mapred.job.queue.name"));
+ assertEquals("sharelib-java",
actionConf.get("oozie.action.sharelib.for.java"));
actionXml = XmlUtils.parseXml("<java>" + "<job-tracker>" +
getJobTrackerUri() + "</job-tracker>" +
"<name-node>" + getNameNodeUri() + "</name-node>
<configuration>" +
@@ -820,4 +822,37 @@ public class TestJavaActionExecutor exte
workflow.setWorkflowInstance(wfInstance);
return workflow;
}
+
+ public void testActionSharelibResolution() throws Exception {
+ JavaActionExecutor ae = new JavaActionExecutor() {
+ @Override
+ protected String getDefaultShareLibName(Element actionXml) {
+ return "java-action-executor";
+ }
+ };
+
+ WorkflowJobBean wfBean = new WorkflowJobBean();
+ String jobConf = "<configuration/>";
+ wfBean.setConf(jobConf);
+
+ WorkflowActionBean action = new WorkflowActionBean();
+ Context context = new Context(wfBean, action);
+
+ Configuration actionConf = new XConfiguration();
+
+ assertEquals("java-action-executor", ae.getShareLibName(context, new
Element("java"), actionConf));
+
+ Services.get().getConf().set("oozie.action.sharelib.for.java",
"java-oozie-conf");
+ assertEquals("java-oozie-conf", ae.getShareLibName(context, new
Element("java"), actionConf));
+
+ jobConf = "<configuration>" + "<property>"
+ + "<name>oozie.action.sharelib.for.java</name>"
+ + "<value>java-job-conf</value>" + "</property>"
+ + "</configuration>";
+ wfBean.setConf(jobConf);
+ assertEquals("java-job-conf", ae.getShareLibName(context, new
Element("java"), actionConf));
+
+ actionConf.set("oozie.action.sharelib.for.java", "java-action-conf");
+ assertEquals("java-action-conf", ae.getShareLibName(context, new
Element("java"), actionConf));
+ }
}
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java?rev=1336477&r1=1336476&r2=1336477&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
Thu May 10 04:08:39 2012
@@ -586,4 +586,12 @@ public class TestMapReduceActionExecutor
String counters = context.getVar("hadoop.counters");
assertTrue(counters.contains("Counter"));
}
+
+ public void testDefaultShareLibName() {
+ MapReduceActionExecutor ae = new MapReduceActionExecutor();
+ Element e = new Element("mapreduce");
+ assertNull(ae.getDefaultShareLibName(e));
+ e.addContent(new Element("streaming"));
+ assertEquals("mapreduce-streaming", ae.getDefaultShareLibName(e));
+ }
}
Modified: incubator/oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki?rev=1336477&r1=1336476&r2=1336477&view=diff
==============================================================================
--- incubator/oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
(original)
+++ incubator/oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki Thu
May 10 04:08:39 2012
@@ -2574,6 +2574,29 @@ A workflow job can specify a share libra
A workflow job can use the system share library by setting the job property
=oozie.use.system.libpath= to =true=.
+---+++ 17.1 Action Share Library Override (since Oozie 3.3)
+
+Oozie share libraries are organized per action type, for example Pig action
share library directory is =share/lib/pig/=
+and Mapreduce Streaming share library direcotry is
=share/library/mapreduce-streaming/=.
+
+Oozie bundles a share library for specific versions of streaming, pig, hive,
sqoop, distcp actions. These versions
+of streaming, pig, hive, sqoop and distcp have been tested and verified to
work correctly with the version of Oozie
+that includes them.
+
+In addition, Oozie provides a mechanism to override the action share library
JARs to allow using an alternate version
+of of the action JARs.
+
+This mechanism enables Oozie administrators to patch share library JARs, to
include alternate versios of the share
+libraries, to provide acess to more than one version at the same time.
+
+The sharelibrary override is supported at server level and at job level. The
share library name is resolved using
+the following precedence order:
+
+ * action.sharelib.for.#ACTIONTYPE# in the action configuration
+ * action.sharelib.for.#ACTIONTYPE# in the job configuration
+ * action.sharelib.for.#ACTIONTYPE# in the oozie server configuration
+ * action's =ActionExecutor getDefaultShareLibName()= method
+
---++ 18 User-Retry for Workflow Actions (since Oozie 3.1)
Oozie provides User-Retry capabilities when an action is in =ERROR= or
=FAILED= state.
Modified: incubator/oozie/trunk/release-log.txt
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1336477&r1=1336476&r2=1336477&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Thu May 10 04:08:39 2012
@@ -1,5 +1,6 @@
-- Oozie trunk
+OOZIE-830 add support for multiple/configurable sharelibs for each action type
(tucu)
OOZIE-773 AuthorizationService should be able to use a group to identify
superusers (tucu)
OOZIE-783 Upgrade to Junit4 (virag via tucu)
OOZIE-812 ooziedb tool does not create VALIDATE_CONN table (tucu)
