Repository: oozie Updated Branches: refs/heads/master c6afe1c11 -> 5ab4c2515
OOZIE-2123 Disable launcher uber mode if classloader options are set (ryota) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/5ab4c251 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/5ab4c251 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/5ab4c251 Branch: refs/heads/master Commit: 5ab4c25156e55d98dce03b68bfb6c74ab805195f Parents: c6afe1c Author: egashira <[email protected]> Authored: Tue Feb 3 16:17:44 2015 -0800 Committer: egashira <[email protected]> Committed: Tue Feb 3 16:17:44 2015 -0800 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 20 ++++++++- .../action/hadoop/TestJavaActionExecutor.java | 46 ++++++++++++++++++++ release-log.txt | 1 + 3 files changed, 66 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/5ab4c251/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index da550a7..7beac5c 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -114,6 +114,8 @@ public class JavaActionExecutor extends ActionExecutor { protected XLog LOG = XLog.getLog(getClass()); private static final Pattern heapPattern = Pattern.compile("-Xmx(([0-9]+)[mMgG])"); public static final String CONF_HADOOP_YARN_UBER_MODE = "oozie.action.launcher." + HADOOP_YARN_UBER_MODE; + public static final String HADOOP_JOB_CLASSLOADER = "mapreduce.job.classloader"; + public static final String HADOOP_USER_CLASSPATH_FIRST = "mapreduce.user.classpath.first"; static { DISALLOWED_PROPERTIES.add(HADOOP_USER); @@ -897,7 +899,12 @@ public class JavaActionExecutor extends ActionExecutor { // setting for uber mode if (launcherJobConf.getBoolean(HADOOP_YARN_UBER_MODE, false)) { - updateConfForUberMode(launcherJobConf); + if (checkPropertiesToDisableUber(launcherJobConf)) { + launcherJobConf.setBoolean(HADOOP_YARN_UBER_MODE, false); + } + else { + updateConfForUberMode(launcherJobConf); + } } // properties from action that are needed by the launcher (e.g. QUEUE NAME, ACLs) @@ -911,6 +918,17 @@ public class JavaActionExecutor extends ActionExecutor { } } + private boolean checkPropertiesToDisableUber(Configuration launcherConf) { + boolean disable = false; + if (launcherConf.getBoolean(HADOOP_JOB_CLASSLOADER, false)) { + disable = true; + } + else if (launcherConf.getBoolean(HADOOP_USER_CLASSPATH_FIRST, false)) { + disable = true; + } + return disable; + } + private void injectCallback(Context context, Configuration conf) { String callback = context.getCallbackUrl("$jobStatus"); if (conf.get("job.end.notification.url") != null) { http://git-wip-us.apache.org/repos/asf/oozie/blob/5ab4c251/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index c18d020..7ee865b 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -1996,6 +1996,52 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals(2560, heapSize); } + public void testDisableUberForProperties() throws Exception { + Element actionXml1 = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + + "<name-node>" + getNameNodeUri() + "</name-node>" + + "<configuration>" + + "<property><name>oozie.launcher.mapreduce.job.classloader</name>" + + "<value>true</value></property>" + + "</configuration>" + + "<main-class>MAIN-CLASS</main-class>" + "</java>"); + JavaActionExecutor ae = new JavaActionExecutor(); + XConfiguration protoConf = new XConfiguration(); + protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); + + WorkflowJobBean wf = createBaseWorkflow(protoConf, "action"); + WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); + action.setType(ae.getType()); + + Context context = new Context(wf, action); + JobConf launcherConf = new JobConf(); + launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml1, launcherConf); + + // uber mode should be disabled since oozie.launcher.mapreduce.job.classloader=true + assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE)); + + Element actionXml2 = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + + "<name-node>" + getNameNodeUri() + "</name-node>" + + "<configuration>" + + "<property><name>oozie.launcher.mapreduce.user.classpath.first</name>" + + "<value>true</value></property>" + + "</configuration>" + + "<main-class>MAIN-CLASS</main-class>" + "</java>"); + ae = new JavaActionExecutor(); + protoConf = new XConfiguration(); + protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); + + wf = createBaseWorkflow(protoConf, "action"); + action = (WorkflowActionBean) wf.getActions().get(0); + action.setType(ae.getType()); + + context = new Context(wf, action); + launcherConf = new JobConf(); + launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml1, launcherConf); + + // uber mode should be disabled since oozie.launcher.mapreduce.job.classloader=true + assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE)); +} + public void testAddToCache() throws Exception { JavaActionExecutor ae = new JavaActionExecutor(); Configuration conf = new XConfiguration(); http://git-wip-us.apache.org/repos/asf/oozie/blob/5ab4c251/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 99b3f7f..2db4bb3 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.2.0 release (trunk - unreleased) +OOZIE-2123 Disable launcher uber mode if classloader options are set (ryota) OOZIE-2118 add createdtime option to workflow jobs query (ryota) OOZIE-2110 cancel delegation token of launcher jobs that stay till child jobs finish (ryota) OOZIE-2119 Distcp action fails when -D option in arguments (ryota)
