Repository: oozie Updated Branches: refs/heads/master 5d26eade7 -> b50d642a8
OOZIE-2474 <job-xml> is not being applied to the launcher job (rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/dec01309 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/dec01309 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/dec01309 Branch: refs/heads/master Commit: dec013093ae5277711a3f59a15d564bcdfb8d35f Parents: 5d26ead Author: Robert Kanter <[email protected]> Authored: Thu Apr 7 17:59:13 2016 -0700 Committer: Robert Kanter <[email protected]> Committed: Thu Apr 7 17:59:13 2016 -0700 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 35 ++++++++--- .../action/hadoop/TestJavaActionExecutor.java | 66 +++++++++++++++----- release-log.txt | 1 + 3 files changed, 78 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/dec01309/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index e055a3d..ff084d1 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -234,7 +234,7 @@ public class JavaActionExecutor extends ActionExecutor { return createBaseHadoopConf(context, actionXml); } - private void injectLauncherProperties(Configuration srcConf, Configuration launcherConf) { + private static void injectLauncherProperties(Configuration srcConf, Configuration launcherConf) { for (Map.Entry<String, String> entry : srcConf) { if (entry.getKey().startsWith("oozie.launcher.")) { String name = entry.getKey().substring("oozie.launcher.".length()); @@ -257,18 +257,20 @@ public class JavaActionExecutor extends ActionExecutor { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); XConfiguration actionDefaultConf = has.createActionDefaultConf(conf.get(HADOOP_JOB_TRACKER), getType()); injectLauncherProperties(actionDefaultConf, launcherConf); - // Inject <configuration> for launcher - Element e = actionXml.getChild("configuration", ns); - if (e != null) { - String strConf = XmlUtils.prettyPrint(e).toString(); - XConfiguration inlineConf = new XConfiguration(new StringReader(strConf)); - injectLauncherProperties(inlineConf, launcherConf); + // Inject <job-xml> and <configuration> for launcher + try { + parseJobXmlAndConfiguration(context, actionXml, appPath, conf, true); + } catch (HadoopAccessorException ex) { + throw convertException(ex); + } catch (URISyntaxException ex) { + throw convertException(ex); } // Inject use uber mode for launcher injectLauncherUseUberMode(launcherConf); XConfiguration.copy(launcherConf, conf); checkForDisallowedProps(launcherConf, "launcher configuration"); - e = actionXml.getChild("config-class", actionXml.getNamespace()); + // Inject config-class for launcher to use for action + Element e = actionXml.getChild("config-class", ns); if (e != null) { conf.set(LauncherMapper.OOZIE_ACTION_CONFIG_CLASS, e.getTextTrim()); } @@ -456,6 +458,11 @@ public class JavaActionExecutor extends ActionExecutor { public static void parseJobXmlAndConfiguration(Context context, Element element, Path appPath, Configuration conf) throws IOException, ActionExecutorException, HadoopAccessorException, URISyntaxException { + parseJobXmlAndConfiguration(context, element, appPath, conf, false); + } + + public static void parseJobXmlAndConfiguration(Context context, Element element, Path appPath, Configuration conf, + boolean isLauncher) throws IOException, ActionExecutorException, HadoopAccessorException, URISyntaxException { Namespace ns = element.getNamespace(); Iterator<Element> it = element.getChildren("job-xml", ns).iterator(); HashMap<String, FileSystem> filesystemsMap = new HashMap<String, FileSystem>(); @@ -494,14 +501,22 @@ public class JavaActionExecutor extends ActionExecutor { context.setErrorInfo("EL_ERROR", ex.getMessage()); } checkForDisallowedProps(jobXmlConf, "job-xml"); - XConfiguration.copy(jobXmlConf, conf); + if (isLauncher) { + injectLauncherProperties(jobXmlConf, conf); + } else { + XConfiguration.copy(jobXmlConf, conf); + } } Element e = element.getChild("configuration", ns); if (e != null) { String strConf = XmlUtils.prettyPrint(e).toString(); XConfiguration inlineConf = new XConfiguration(new StringReader(strConf)); checkForDisallowedProps(inlineConf, "inline configuration"); - XConfiguration.copy(inlineConf, conf); + if (isLauncher) { + injectLauncherProperties(inlineConf, conf); + } else { + XConfiguration.copy(inlineConf, conf); + } } } http://git-wip-us.apache.org/repos/asf/oozie/blob/dec01309/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index 35390c4..d2f90c6 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -184,6 +184,9 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { OutputStream os = getFileSystem().create(new Path(getFsTestCaseDir(), "job.xml")); conf.writeXml(os); os.close(); + os = getFileSystem().create(new Path(getFsTestCaseDir(), new Path("app", "job.xml"))); + conf.writeXml(os); + os.close(); conf = new XConfiguration(); conf.set("e", "E"); @@ -191,6 +194,9 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { os = getFileSystem().create(new Path(getFsTestCaseDir(), "job2.xml")); conf.writeXml(os); os.close(); + os = getFileSystem().create(new Path(getFsTestCaseDir(), new Path("app", "job2.xml"))); + conf.writeXml(os); + os.close(); conf = ae.createBaseHadoopConf(context, actionXml); assertEquals(protoConf.get(WorkflowAppService.HADOOP_USER), conf.get(WorkflowAppService.HADOOP_USER)); @@ -202,8 +208,10 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals("LA", conf.get("oozie.launcher.a")); assertEquals("LA", conf.get("a")); assertNull(conf.get("b")); - assertNull(conf.get("oozie.launcher.d")); - assertNull(conf.get("d")); + assertEquals("D", conf.get("oozie.launcher.d")); + assertEquals("D", conf.get("d")); + assertEquals("F", conf.get("oozie.launcher.f")); + assertEquals("F", conf.get("f")); assertNull(conf.get("action.foo")); assertEquals("action.barbar", conf.get("action.foofoo")); @@ -214,8 +222,10 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals("BB", conf.get("b")); assertEquals("C", conf.get("c")); assertEquals("D", conf.get("oozie.launcher.d")); + assertNull(conf.get("d")); assertEquals("E", conf.get("e")); assertEquals("F", conf.get("oozie.launcher.f")); + assertNull(conf.get("f")); assertEquals("action.bar", conf.get("action.foo")); conf = ae.createBaseHadoopConf(context, actionXml); @@ -1216,7 +1226,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { public void testJavaOpts() throws Exception { String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" - + getNameNodeUri() + "</name-node>" + "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" + + getNameNodeUri() + "</name-node>" + "<configuration>" + "<property><name>oozie.launcher.a</name><value>LA</value></property>" + "<property><name>a</name><value>AA</value></property>" + "<property><name>b</name><value>BB</value></property>" + "</configuration>" @@ -1242,7 +1252,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2", conf.get("mapreduce.map.java.opts")); actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" - + getNameNodeUri() + "</name-node>" + "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" + + getNameNodeUri() + "</name-node>" + "<configuration>" + "<property><name>oozie.launcher.a</name><value>LA</value></property>" + "<property><name>a</name><value>AA</value></property>" + "<property><name>b</name><value>BB</value></property>" + "</configuration>" @@ -1267,7 +1277,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2", conf.get("mapreduce.map.java.opts")); actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" - + getNameNodeUri() + "</name-node>" + "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" + + getNameNodeUri() + "</name-node>" + "<configuration>" + "<property><name>oozie.launcher.a</name><value>LA</value></property>" + "<property><name>a</name><value>AA</value></property>" + "<property><name>b</name><value>BB</value></property>" @@ -1294,7 +1304,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals("JAVA-OPT3 JAVA-OPT1 JAVA-OPT2", conf.get("mapreduce.map.java.opts")); actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" - + getNameNodeUri() + "</name-node>" + "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" + + getNameNodeUri() + "</name-node>" + "<configuration>" + "<property><name>oozie.launcher.a</name><value>LA</value></property>" + "<property><name>a</name><value>AA</value></property>" + "<property><name>b</name><value>BB</value></property>" @@ -1321,7 +1331,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals("-Xmx200m JAVA-OPT3 JAVA-OPT1 JAVA-OPT2", conf.get("mapreduce.map.java.opts")); actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" - + getNameNodeUri() + "</name-node>" + "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" + + getNameNodeUri() + "</name-node>" + "<configuration>" + "<property><name>oozie.launcher.a</name><value>LA</value></property>" + "<property><name>a</name><value>AA</value></property>" + "<property><name>b</name><value>BB</value></property>" @@ -1359,8 +1369,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { getFileSystem().create(jar2Path).close(); String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + - "<name-node>" + getNameNodeUri() + "</name-node>" + - "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" + "<configuration>" + + "<name-node>" + getNameNodeUri() + "</name-node>" + "<configuration>" + "<property><name>oozie.launcher.oozie.libpath</name><value>" + actionLibPath + "</value></property>" + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" + "</java>"; @@ -1383,8 +1392,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { getFileSystem().create(jar3Path).close(); actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + - "<name-node>" + getNameNodeUri() + "</name-node>" + - "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" + "<configuration>" + + "<name-node>" + getNameNodeUri() + "</name-node>" + "<configuration>" + "<property><name>oozie.launcher.oozie.libpath</name><value>" + jar3Path + "</value></property>" + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" + "</java>"; @@ -1403,8 +1411,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { // Test adding a directory and a file (comma separated) actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + - "<name-node>" + getNameNodeUri() + "</name-node>" + - "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" + "<configuration>" + + "<name-node>" + getNameNodeUri() + "</name-node>" + "<configuration>" + "<property><name>oozie.launcher.oozie.libpath</name><value>" + actionLibPath + "," + jar3Path + "</value></property>" + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" + @@ -1454,7 +1461,6 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + - "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" + "<main-class>MAIN-CLASS</main-class>" + "</java>"; Element eActionXml = XmlUtils.parseXml(actionXml); @@ -1782,6 +1788,38 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals("test", conf.get("user")); } + public void testJobXmlWithOozieLauncher() throws Exception { + String str = "<java>" + + "<job-xml>job.xml</job-xml>" + + "<configuration>" + + "<property><name>oozie.launcher.p2</name><value>v2b</value></property>" + + "<property><name>p4</name><value>v4</value></property>" + + "</configuration>" + + "</java>"; + Element xml = XmlUtils.parseXml(str); + Path appPath = new Path(getFsTestCaseDir(), "app"); + getFileSystem().mkdirs(appPath); + + XConfiguration jConf = new XConfiguration(); + jConf.set("oozie.launcher.p1", "v1"); + jConf.set("oozie.launcher.p2", "v2a"); + jConf.set("p3", "v3"); + OutputStream os = getFileSystem().create(new Path(appPath, "job.xml")); + jConf.writeXml(os); + os.close(); + + Configuration conf = new XConfiguration(); + assertEquals(0, conf.size()); + JavaActionExecutor jae = new JavaActionExecutor("java"); + jae.setupLauncherConf(conf, xml, appPath, createContext("<java/>", null)); + assertEquals(5, conf.size()); + assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); + assertEquals("v1", conf.get("oozie.launcher.p1")); + assertEquals("v1", conf.get("p1")); + assertEquals("v2b", conf.get("oozie.launcher.p2")); + assertEquals("v2b", conf.get("p2")); + } + public void testInjectLauncherUseUberMode() throws Exception { // default -- should set to true JavaActionExecutor jae = new JavaActionExecutor(); http://git-wip-us.apache.org/repos/asf/oozie/blob/dec01309/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index b8f7b03..b4e2e75 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2474 <job-xml> is not being applied to the launcher job (rkanter) OOZIE-2486 TestSLAEventsGetForFilterJPAExecutor is flakey (rkanter) OOZIE-2481 Add YARN_CONF_DIR in the Shell action (harsh) OOZIE-2492 JSON security issue in js code (fdenes via rkanter)
