Repository: oozie Updated Branches: refs/heads/master 2fc2fc9ec -> a40535869
OOZIE-1385 Make Uber Mode the default (rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a4053586 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a4053586 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a4053586 Branch: refs/heads/master Commit: a40535869f406a9ca286565bd3c49a6d6a3931a6 Parents: 2fc2fc9 Author: Robert Kanter <[email protected]> Authored: Mon Nov 10 13:00:08 2014 -0800 Committer: Robert Kanter <[email protected]> Committed: Mon Nov 10 13:00:08 2014 -0800 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 16 ++++- core/src/main/resources/oozie-default.xml | 21 +++++- .../action/hadoop/TestJavaActionExecutor.java | 75 ++++++++++++++++---- .../action/hadoop/TestShellActionExecutor.java | 2 +- .../oozie/service/TestConfigurationService.java | 4 +- release-log.txt | 1 + 6 files changed, 96 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/a4053586/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 7349d3f..687a948 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -259,10 +259,20 @@ public class JavaActionExecutor extends ActionExecutor { } void injectLauncherUseUberMode(Configuration launcherConf) { - // Set Uber Mode for the launcher (YARN only, ignored by MR1) if not set by action conf and not disabled in oozie-site + // Set Uber Mode for the launcher (YARN only, ignored by MR1) + // Priority: + // 1. action's <configuration> + // 2. oozie.action.#action-type#.launcher.mapreduce.job.ubertask.enable + // 3. oozie.action.launcher.mapreduce.job.ubertask.enable if (launcherConf.get(HADOOP_YARN_UBER_MODE) == null) { - if (ConfigurationService.getBoolean(getOozieConf(), CONF_HADOOP_YARN_UBER_MODE)) { - launcherConf.setBoolean(HADOOP_YARN_UBER_MODE, true); + if (ConfigurationService.get("oozie.action." + getType() + ".launcher." + HADOOP_YARN_UBER_MODE).length() > 0) { + if (ConfigurationService.getBoolean("oozie.action." + getType() + ".launcher." + HADOOP_YARN_UBER_MODE)) { + launcherConf.setBoolean(HADOOP_YARN_UBER_MODE, true); + } + } else { + if (ConfigurationService.getBoolean("oozie.action.launcher." + HADOOP_YARN_UBER_MODE)) { + launcherConf.setBoolean(HADOOP_YARN_UBER_MODE, true); + } } } } http://git-wip-us.apache.org/repos/asf/oozie/blob/a4053586/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 17155a1..19cae9d 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -1622,11 +1622,26 @@ <property> <name>oozie.action.launcher.mapreduce.job.ubertask.enable</name> + <value>true</value> + <description> + Enables Uber Mode for the launcher job in YARN/Hadoop 2 (no effect in Hadoop 1) for all action types by default. + This can be overridden on a per-action-type basis by setting + oozie.action.#action-type#.launcher.mapreduce.job.ubertask.enable in oozie-site.xml (where #action-type# is the action + type; for example, "pig"). And that can be overridden on a per-action basis by setting + oozie.launcher.mapreduce.job.ubertask.enable in an action's configuration section in a workflow. In summary, the + priority is this: + 1. action's configuration section in a workflow + 2. oozie.action.#action-type#.launcher.mapreduce.job.ubertask.enable in oozie-site + 3. oozie.action.launcher.mapreduce.job.ubertask.enable in oozie-site + </description> + </property> + + <property> + <name>oozie.action.shell.launcher.mapreduce.job.ubertask.enable</name> <value>false</value> <description> - Enables Uber Mode for the launcher job in YARN/Hadoop 2 (no effect in Hadoop 1). - Setting oozie.launcher.mapreduce.job.ubertask.enable in a an action's configuration section overrides this for that - action. + The Shell action may have issues with the $PATH environment when using Uber Mode, and so Uber Mode is disabled by + default for it. See oozie.action.launcher.mapreduce.job.ubertask.enable </description> </property> http://git-wip-us.apache.org/repos/asf/oozie/blob/a4053586/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index 343a31e..48166a5 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -1614,11 +1614,6 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { } public void testInjectLauncherUseUberMode() throws Exception { - // TODO: Delete these two lines once uber mode is set back to the - // default (OOZIE-1385) - assertFalse(Services.get().getConf().getBoolean("oozie.action.launcher.mapreduce.job.ubertask.enable", true)); - Services.get().getConf().setBoolean("oozie.action.launcher.mapreduce.job.ubertask.enable", true); - // default -- should set to true JavaActionExecutor jae = new JavaActionExecutor(); Configuration conf = new Configuration(false); @@ -1640,10 +1635,56 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { jae.injectLauncherUseUberMode(conf); assertEquals("false", conf.get("mapreduce.job.ubertask.enable")); - // disable at oozie-site level (default is to be enabled) -- redo above - // tests + // disable at oozie-site level for just the "test" action + Services.get().getConf().setBoolean("oozie.action.test.launcher.mapreduce.job.ubertask.enable", false); + JavaActionExecutor tjae = new JavaActionExecutor("test"); + + // default -- should not set + conf = new Configuration(false); + assertNull(conf.get("mapreduce.job.ubertask.enable")); + tjae.injectLauncherUseUberMode(conf); + assertNull(conf.get("mapreduce.job.ubertask.enable")); + // default -- should be true + conf = new Configuration(false); + assertNull(conf.get("mapreduce.job.ubertask.enable")); + jae.injectLauncherUseUberMode(conf); + assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); + + // action conf set to true -- should keep at true + conf = new Configuration(false); + assertNull(conf.get("mapreduce.job.ubertask.enable")); + conf.setBoolean("mapreduce.job.ubertask.enable", true); + tjae.injectLauncherUseUberMode(conf); + assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); + // action conf set to true -- should keep at true + conf = new Configuration(false); + assertNull(conf.get("mapreduce.job.ubertask.enable")); + conf.setBoolean("mapreduce.job.ubertask.enable", true); + jae.injectLauncherUseUberMode(conf); + assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); + + // action conf set to false -- should keep at false + conf = new Configuration(false); + assertNull(conf.get("mapreduce.job.ubertask.enable")); + conf.setBoolean("mapreduce.job.ubertask.enable", false); + tjae.injectLauncherUseUberMode(conf); + assertEquals("false", conf.get("mapreduce.job.ubertask.enable")); + // action conf set to false -- should keep at false + conf = new Configuration(false); + assertNull(conf.get("mapreduce.job.ubertask.enable")); + conf.setBoolean("mapreduce.job.ubertask.enable", false); + jae.injectLauncherUseUberMode(conf); + assertEquals("false", conf.get("mapreduce.job.ubertask.enable")); + + // disable at oozie-site level for all actions except for the "test" action + Services.get().getConf().setBoolean("oozie.action.test.launcher.mapreduce.job.ubertask.enable", true); Services.get().getConf().setBoolean("oozie.action.launcher.mapreduce.job.ubertask.enable", false); + // default -- should be true + conf = new Configuration(false); + assertNull(conf.get("mapreduce.job.ubertask.enable")); + tjae.injectLauncherUseUberMode(conf); + assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); // default -- should not set conf = new Configuration(false); assertNull(conf.get("mapreduce.job.ubertask.enable")); @@ -1654,6 +1695,12 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { conf = new Configuration(false); assertNull(conf.get("mapreduce.job.ubertask.enable")); conf.setBoolean("mapreduce.job.ubertask.enable", true); + tjae.injectLauncherUseUberMode(conf); + assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); + // action conf set to true -- should keep at true + conf = new Configuration(false); + assertNull(conf.get("mapreduce.job.ubertask.enable")); + conf.setBoolean("mapreduce.job.ubertask.enable", true); jae.injectLauncherUseUberMode(conf); assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); @@ -1661,14 +1708,17 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { conf = new Configuration(false); assertNull(conf.get("mapreduce.job.ubertask.enable")); conf.setBoolean("mapreduce.job.ubertask.enable", false); + tjae.injectLauncherUseUberMode(conf); + assertEquals("false", conf.get("mapreduce.job.ubertask.enable")); + // action conf set to false -- should keep at false + conf = new Configuration(false); + assertNull(conf.get("mapreduce.job.ubertask.enable")); + conf.setBoolean("mapreduce.job.ubertask.enable", false); jae.injectLauncherUseUberMode(conf); assertEquals("false", conf.get("mapreduce.job.ubertask.enable")); } public void testUpdateConfForUberMode() throws Exception { - - Services.get().getConf().setBoolean("oozie.action.launcher.mapreduce.job.ubertask.enable", true); - Element actionXml1 = XmlUtils .parseXml("<java>" + "<job-tracker>" @@ -1790,9 +1840,6 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { } public void testUpdateConfForUberModeWithEnvDup() throws Exception { - - Services.get().getConf().setBoolean("oozie.action.launcher.mapreduce.job.ubertask.enable", true); - Element actionXml1 = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<configuration>" + "<property><name>oozie.launcher.yarn.app.mapreduce.am.env</name>" @@ -1859,8 +1906,6 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { } public void testUpdateConfForUberModeForJavaOpts() throws Exception { - Services.get().getConf().setBoolean("oozie.action.launcher.mapreduce.job.ubertask.enable", true); - Element actionXml1 = XmlUtils .parseXml("<java>" + "<job-tracker>" http://git-wip-us.apache.org/repos/asf/oozie/blob/a4053586/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java index 7795963..2775baa 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java @@ -326,7 +326,7 @@ public class TestShellActionExecutor extends ActionExecutorTestCase { } public void testShellMainPathInUber() throws Exception { - Services.get().getConf().setBoolean("oozie.action.launcher.mapreduce.job.ubertask.enable", true); + Services.get().getConf().setBoolean("oozie.action.shell.launcher.mapreduce.job.ubertask.enable", true); Element actionXml = XmlUtils.parseXml("<shell>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<exec>script.sh</exec>" http://git-wip-us.apache.org/repos/asf/oozie/blob/a4053586/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java b/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java index d09bfc0..b1dde2c 100644 --- a/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java +++ b/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java @@ -208,7 +208,9 @@ public class TestConfigurationService extends XTestCase { assertEquals(2048, ConfigurationService.getInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA)); assertEquals("http://localhost:8080/oozie?job=";, ConfigurationService.get(JobXCommand.CONF_CONSOLE_URL)); - assertEquals(false, ConfigurationService.getBoolean(JavaActionExecutor.CONF_HADOOP_YARN_UBER_MODE)); + assertEquals(true, ConfigurationService.getBoolean(JavaActionExecutor.CONF_HADOOP_YARN_UBER_MODE)); + assertEquals(false, ConfigurationService.getBoolean( + "oozie.action.shell.launcher." + JavaActionExecutor.HADOOP_YARN_UBER_MODE)); assertEquals(false, ConfigurationService.getBoolean(HadoopAccessorService.KERBEROS_AUTH_ENABLED)); assertEquals(0, ConfigurationService.getStrings("no.defined").length); http://git-wip-us.apache.org/repos/asf/oozie/blob/a4053586/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index a57365e..aa8ef33 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.2.0 release (trunk - unreleased) +OOZIE-1385 Make Uber Mode the default (rkanter) OOZIE-1890 Make oozie-site empty and reconcile defaults between oozie-default and the code (seoeun25 via rkanter) OOZIE-2001 Workflow re-runs doesn't update coord action status (jaydeepvishwakarma via shwethags) OOZIE-2048 HadoopAccessorService should also process ssl_client.xml (venkatnrangan via bzhang)
