Author: rkanter
Date: Fri Feb 15 06:10:32 2013
New Revision: 1446454
URL: http://svn.apache.org/r1446454
Log:
OOZIE-1226 Workflow lib path not found in classpath for a subworkflow (rkanter)
Modified:
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
oozie/branches/branch-3.3/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
oozie/branches/branch-3.3/release-log.txt
Modified:
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
URL:
http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java?rev=1446454&r1=1446453&r2=1446454&view=diff
==============================================================================
---
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
(original)
+++
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
Fri Feb 15 06:10:32 2013
@@ -37,8 +37,11 @@ import java.io.Reader;
import java.io.StringWriter;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedHashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -57,8 +60,13 @@ public abstract class WorkflowAppService
public static final String CONFG_MAX_WF_LENGTH = CONF_PREFIX +
"WorkflowDefinitionMaxLength";
+ public static final String OOZIE_SUBWORKFLOW_CLASSPATH_INHERITANCE =
"oozie.subworkflow.classpath.inheritance";
+
+ public static final String OOZIE_WF_SUBWORKFLOW_CLASSPATH_INHERITANCE =
"oozie.wf.subworkflow.classpath.inheritance";
+
private Path systemLibPath;
private long maxWFLength;
+ private boolean oozieSubWfCPInheritance;
/**
* Initialize the workflow application service.
@@ -74,6 +82,8 @@ public abstract class WorkflowAppService
}
maxWFLength = conf.getInt(CONFG_MAX_WF_LENGTH, 100000);
+
+ oozieSubWfCPInheritance =
conf.getBoolean(OOZIE_SUBWORKFLOW_CLASSPATH_INHERITANCE, true);
}
/**
@@ -195,6 +205,30 @@ public abstract class WorkflowAppService
}
}
+ // Check if a subworkflow should inherit the libs from the parent
WF
+ // OOZIE_WF_SUBWORKFLOW_CLASSPATH_INHERITANCE has priority over
OOZIE_SUBWORKFLOW_CLASSPATH_INHERITANCE from oozie-site
+ // If OOZIE_WF_SUBWORKFLOW_CLASSPATH_INHERITANCE isn't specified,
we use OOZIE_SUBWORKFLOW_CLASSPATH_INHERITANCE
+ if (jobConf.getBoolean(OOZIE_WF_SUBWORKFLOW_CLASSPATH_INHERITANCE,
oozieSubWfCPInheritance)) {
+ // Keep any libs from a parent workflow that might already be
in APP_LIB_PATH_LIST and also remove duplicates
+ String[] parentFilePaths =
jobConf.getStrings(APP_LIB_PATH_LIST);
+ if (parentFilePaths != null && parentFilePaths.length > 0) {
+ String[] filePathsNames = filePaths.toArray(new
String[filePaths.size()]);
+ for (int i = 0; i < filePathsNames.length; i++) {
+ Path p = new Path(filePathsNames[i]);
+ filePathsNames[i] = p.getName();
+ }
+ Arrays.sort(filePathsNames);
+ List<String> nonDuplicateParentFilePaths = new
ArrayList<String>();
+ for (String parentFilePath : parentFilePaths) {
+ Path p = new Path(parentFilePath);
+ if (Arrays.binarySearch(filePathsNames, p.getName()) <
0) {
+ nonDuplicateParentFilePaths.add(parentFilePath);
+ }
+ }
+ filePaths.addAll(nonDuplicateParentFilePaths);
+ }
+ }
+
conf.setStrings(APP_LIB_PATH_LIST, filePaths.toArray(new
String[filePaths.size()]));
//Add all properties start with 'oozie.'
Modified:
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
URL:
http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java?rev=1446454&r1=1446453&r2=1446454&view=diff
==============================================================================
---
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
(original)
+++
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
Fri Feb 15 06:10:32 2013
@@ -35,10 +35,12 @@ import java.io.FileWriter;
import java.io.Reader;
import java.io.Writer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import junit.framework.Assert;
+import org.apache.hadoop.fs.Path;
import org.apache.oozie.workflow.lite.StartNodeDef;
public class TestLiteWorkflowAppService extends XTestCase {
@@ -452,46 +454,191 @@ public class TestLiteWorkflowAppService
}
}
- public void
testCreateprotoConfWithSubWorkflow_CheckSubworkflowLibInClasspath() throws
Exception {
- // When subworkflow has non-empty lib directory, APP_LIB_PATH_LIST for
- // subworkflow should maintain its corresponding libraries in path
+ private static String[] parentLibs1 = {"parent1.jar", "parent2.jar"};
+ private static String[] childLibs1 = {"child1.jar", "child2.so"};
+ private static String[] parentLibs2 = {"parent1.jar", "parent2.jar"};
+ private static String[] childLibs2 = {};;
+ private static String[] parentLibs3 = {};;
+ private static String[] childLibs3 = {"child1.jar", "child2.so"};;
+ private static String[] parentLibs4 = {};;
+ private static String[] childLibs4 = {};;
+ private static String[] parentLibs5 = {"parent1.jar", "parent2.jar",
"same.jar"};;
+ private static String[] childLibs5 = {"child1.jar", "same.jar",
"child2.so"};;
+
+ public void testCreateProtoConfWithSubWorkflowLib1() throws Exception {
+ String inherit = "true";
+ String inheritWF = null;
+
+ String[] expectedLibs1 = {"parent1.jar", "parent2.jar", "child1.jar",
"child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 1, parentLibs1,
childLibs1, expectedLibs1);
+
+ String[] expectedLibs2 = {"parent1.jar", "parent2.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 2, parentLibs2,
childLibs2, expectedLibs2);
+
+ String[] expectedLibs3 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 3, parentLibs3,
childLibs3, expectedLibs3);
+
+ String[] expectedLibs4 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 4, parentLibs4,
childLibs4, expectedLibs4);
+
+ String[] expectedLibs5 = {"parent1.jar", "parent2.jar", "child1.jar",
"child2.so", "same.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 5, parentLibs5,
childLibs5, expectedLibs5);
+ }
+
+ public void testCreateProtoConfWithSubWorkflowLib2() throws Exception {
+ String inherit = "false";
+ String inheritWF = null;
+
+ String[] expectedLibs1 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 1, parentLibs1,
childLibs1, expectedLibs1);
+
+ String[] expectedLibs2 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 2, parentLibs2,
childLibs2, expectedLibs2);
+
+ String[] expectedLibs3 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 3, parentLibs3,
childLibs3, expectedLibs3);
+
+ String[] expectedLibs4 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 4, parentLibs4,
childLibs4, expectedLibs4);
+
+ String[] expectedLibs5 = {"child1.jar", "child2.so", "same.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 5, parentLibs5,
childLibs5, expectedLibs5);
+ }
+
+ public void testCreateProtoConfWithSubWorkflowLib3() throws Exception {
+ String inherit = "true";
+ String inheritWF = "true";
+
+ String[] expectedLibs1 = {"parent1.jar", "parent2.jar", "child1.jar",
"child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 1, parentLibs1,
childLibs1, expectedLibs1);
+
+ String[] expectedLibs2 = {"parent1.jar", "parent2.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 2, parentLibs2,
childLibs2, expectedLibs2);
+
+ String[] expectedLibs3 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 3, parentLibs3,
childLibs3, expectedLibs3);
+
+ String[] expectedLibs4 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 4, parentLibs4,
childLibs4, expectedLibs4);
+
+ String[] expectedLibs5 = {"parent1.jar", "parent2.jar", "child1.jar",
"child2.so", "same.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 5, parentLibs5,
childLibs5, expectedLibs5);
+ }
+
+ public void testCreateProtoConfWithSubWorkflowLib4() throws Exception {
+ String inherit = "false";
+ String inheritWF = "true";
+
+ String[] expectedLibs1 = {"parent1.jar", "parent2.jar", "child1.jar",
"child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 1, parentLibs1,
childLibs1, expectedLibs1);
+
+ String[] expectedLibs2 = {"parent1.jar", "parent2.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 2, parentLibs2,
childLibs2, expectedLibs2);
+
+ String[] expectedLibs3 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 3, parentLibs3,
childLibs3, expectedLibs3);
+
+ String[] expectedLibs4 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 4, parentLibs4,
childLibs4, expectedLibs4);
+
+ String[] expectedLibs5 = {"parent1.jar", "parent2.jar", "child1.jar",
"child2.so", "same.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 5, parentLibs5,
childLibs5, expectedLibs5);
+ }
+
+ public void testCreateProtoConfWithSubWorkflowLib5() throws Exception {
+ String inherit = "true";
+ String inheritWF = "false";
+
+ String[] expectedLibs1 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 1, parentLibs1,
childLibs1, expectedLibs1);
+
+ String[] expectedLibs2 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 2, parentLibs2,
childLibs2, expectedLibs2);
+
+ String[] expectedLibs3 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 3, parentLibs3,
childLibs3, expectedLibs3);
+
+ String[] expectedLibs4 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 4, parentLibs4,
childLibs4, expectedLibs4);
+
+ String[] expectedLibs5 = {"child1.jar", "child2.so", "same.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 5, parentLibs5,
childLibs5, expectedLibs5);
+ }
+
+ public void testCreateProtoConfWithSubWorkflowLib6() throws Exception {
+ String inherit = "false";
+ String inheritWF = "false";
+
+ String[] expectedLibs1 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 1, parentLibs1,
childLibs1, expectedLibs1);
+
+ String[] expectedLibs2 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 2, parentLibs2,
childLibs2, expectedLibs2);
+
+ String[] expectedLibs3 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 3, parentLibs3,
childLibs3, expectedLibs3);
+
+ String[] expectedLibs4 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 4, parentLibs4,
childLibs4, expectedLibs4);
+
+ String[] expectedLibs5 = {"child1.jar", "child2.so", "same.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 5, parentLibs5,
childLibs5, expectedLibs5);
+ }
+
+ public void checkSubworkflowLibHelper(String inherit, String inheritWF,
int unique, String[] parentLibs, String[] childLibs,
+ String[] expectedLibs) throws Exception {
Services services = new Services();
try {
+ services.getConf().set("oozie.subworkflow.classpath.inheritance",
inherit);
services.init();
Reader reader = IOUtils.getResourceAsReader("wf-schema-valid.xml",
-1);
- Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
+ String childWFDir = createTestCaseSubDir("child-wf-" + unique);
+ Writer writer = new FileWriter(childWFDir + File.separator +
"workflow.xml");
IOUtils.copyCharStream(reader, writer);
- createTestCaseSubDir("lib");
- writer = new FileWriter(getTestCaseDir() +
"/lib/childdependency1.jar");
- writer.write("bla bla");
- writer.close();
- writer = new FileWriter(getTestCaseDir() +
"/lib/childdependency2.so");
- writer.write("bla bla");
- writer.close();
WorkflowAppService wps =
Services.get().get(WorkflowAppService.class);
Configuration jobConf = new XConfiguration();
- jobConf.set(OozieClient.APP_PATH, "file://" + getTestCaseDir() +
File.separator + "workflow.xml");
+ jobConf.set(OozieClient.APP_PATH, "file://" + childWFDir +
File.separator + "workflow.xml");
jobConf.set(OozieClient.USER_NAME, getTestUser());
- jobConf.set(WorkflowAppService.APP_LIB_PATH_LIST,
"parentdependency1.jar");
+ if (inheritWF != null) {
+ jobConf.set("oozie.wf.subworkflow.classpath.inheritance",
inheritWF);
+ }
+
+ String childLibDir = createTestCaseSubDir("child-wf-" + unique +
File.separator + "lib");
+ for (String childLib : childLibs) {
+ writer = new FileWriter(childLibDir + File.separator +
childLib);
+ writer.write("bla bla");
+ writer.close();
+ }
+ String parentWFDir = createTestCaseSubDir("parent-wf-" + unique);
+ String parentLibDir = createTestCaseSubDir("parent-wf-" + unique +
File.separator + "lib");
+ String[] parentLibsFullPaths = new String[parentLibs.length];
+ for (int i = 0; i < parentLibs.length; i++) {
+ parentLibsFullPaths[i] = parentLibDir + File.separator +
parentLibs[i];
+ writer = new FileWriter(parentLibsFullPaths[i]);
+ writer.write("bla bla");
+ writer.close();
+ }
+ // Set the parent libs
+ jobConf.setStrings(WorkflowAppService.APP_LIB_PATH_LIST,
parentLibsFullPaths);
Configuration protoConf = wps.createProtoActionConf(jobConf,
"authToken", true);
assertEquals(getTestUser(), protoConf.get(OozieClient.USER_NAME));
- assertEquals(2,
protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST).length);
- String f1 =
protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[0];
- String f2 =
protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[1];
- String ref1 = "file://" + getTestCaseDir() +
"/lib/childdependency1.jar";
- String ref2 = "file://" + getTestCaseDir() +
"/lib/childdependency2.so";
- List<String> expected = new ArrayList<String>();
- expected.add(ref1);
- expected.add(ref2);
- List<String> found = new ArrayList<String>();
- found.add(f1);
- found.add(f2);
- Collections.sort(found);
- Collections.sort(expected);
- assertEquals(expected, found);
+ String[] foundLibs =
protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST);
+ if (expectedLibs.length > 0) {
+ assertEquals(expectedLibs.length, foundLibs.length);
+ for (int i = 0; i < foundLibs.length; i++) {
+ Path p = new Path(foundLibs[i]);
+ foundLibs[i] = p.getName();
+ }
+ Arrays.sort(expectedLibs);
+ Arrays.sort(foundLibs);
+ assertEquals(Arrays.toString(expectedLibs),
Arrays.toString(foundLibs));
+ }
+ else {
+ assertEquals(null, foundLibs);
+ }
}
finally {
services.destroy();
Modified:
oozie/branches/branch-3.3/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
URL:
http://svn.apache.org/viewvc/oozie/branches/branch-3.3/docs/src/site/twiki/WorkflowFunctionalSpec.twiki?rev=1446454&r1=1446453&r2=1446454&view=diff
==============================================================================
--- oozie/branches/branch-3.3/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
(original)
+++ oozie/branches/branch-3.3/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
Fri Feb 15 06:10:32 2013
@@ -1280,6 +1280,12 @@ In the above example, the workflow defin
A configuration parameter =input.dir= is being passed as job property to the
child workflow job.
+The subworkflow can inherit the lib jars from the parent workflow by setting
=oozie.subworkflow.classpath.inheritance= to true
+in oozie-site.xml or on a per-job basis by setting
=oozie.wf.subworkflow.classpath.inheritance= to true in a job.properties file.
+If both are specified, =oozie.wf.subworkflow.classpath.inheritance= has
priority. If the subworkflow and the parent have
+conflicting jars, the subworkflow's jar has priority. By default,
=oozie.wf.subworkflow.classpath.inheritance= is set to true.
+
+
#JavaAction
---++++ 3.2.7 Java Action
Modified: oozie/branches/branch-3.3/release-log.txt
URL:
http://svn.apache.org/viewvc/oozie/branches/branch-3.3/release-log.txt?rev=1446454&r1=1446453&r2=1446454&view=diff
==============================================================================
--- oozie/branches/branch-3.3/release-log.txt (original)
+++ oozie/branches/branch-3.3/release-log.txt Fri Feb 15 06:10:32 2013
@@ -12,6 +12,7 @@ OOZIE-944 Implement Workflow Generator U
-- Oozie 3.3.2 (unreleased)
+OOZIE-1226 Workflow lib path not found in classpath for a subworkflow (rkanter)
OOZIE-1184 Demo example job.properties has an unused parameter (udai via
rkanter)
OOZIE-1211 oozie does not support duplicated dataset (jaoki via virag)
OOZIE-1187 reduce memory usage of SLA query (invoked by CLI command) to avoid
OOM (egashira via virag)
