Author: rkanter
Date: Thu May 2 17:34:39 2013
New Revision: 1478460
URL: http://svn.apache.org/r1478460
Log:
OOZIE-1335 The launcher job should use uber mode in Hadoop 2 by default
(rkanter)
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
oozie/trunk/core/src/main/resources/oozie-default.xml
oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
oozie/trunk/release-log.txt
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java?rev=1478460&r1=1478459&r2=1478460&view=diff
==============================================================================
---
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
(original)
+++
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
Thu May 2 17:34:39 2013
@@ -90,6 +90,7 @@ public class JavaActionExecutor extends
public final static String MAX_EXTERNAL_STATS_SIZE =
"oozie.external.stats.max.size";
public static final String ACL_VIEW_JOB = "mapreduce.job.acl-view-job";
public static final String ACL_MODIFY_JOB = "mapreduce.job.acl-modify-job";
+ private static final String HADOOP_YARN_UBER_MODE =
"mapreduce.job.ubertask.enable";
private static int maxActionOutputLen;
private static int maxExternalStatsSize;
@@ -239,6 +240,7 @@ public class JavaActionExecutor extends
XConfiguration actionDefaultConf =
has.createActionDefaultConf(conf.get(HADOOP_JOB_TRACKER), getType());
injectLauncherProperties(actionDefaultConf, launcherConf);
injectLauncherProperties(inlineConf, launcherConf);
+ injectLauncherUseUberMode(launcherConf);
checkForDisallowedProps(launcherConf, "launcher
configuration");
XConfiguration.copy(launcherConf, conf);
}
@@ -249,6 +251,15 @@ public class JavaActionExecutor extends
}
}
+ void injectLauncherUseUberMode(Configuration launcherConf) {
+ // Set Uber Mode for the launcher (YARN only, ignored by MR1) if not
set by action conf and not disabled in oozie-site
+ if (launcherConf.get(HADOOP_YARN_UBER_MODE) == null) {
+ if
(getOozieConf().getBoolean("oozie.action.launcher.mapreduce.job.ubertask.enable",
true)) {
+ launcherConf.setBoolean(HADOOP_YARN_UBER_MODE, true);
+ }
+ }
+ }
+
public static void parseJobXmlAndConfiguration(Context context, Element
element, Path appPath, Configuration conf)
throws IOException, ActionExecutorException,
HadoopAccessorException, URISyntaxException {
Namespace ns = element.getNamespace();
Modified: oozie/trunk/core/src/main/resources/oozie-default.xml
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/resources/oozie-default.xml?rev=1478460&r1=1478459&r2=1478460&view=diff
==============================================================================
--- oozie/trunk/core/src/main/resources/oozie-default.xml (original)
+++ oozie/trunk/core/src/main/resources/oozie-default.xml Thu May 2 17:34:39
2013
@@ -1424,6 +1424,19 @@
The frequency in seconds at which the PauseTransitService will run.
</description>
</property>
+
+ <!-- JavaActionExecutor -->
+ <!-- This is common to the subclasses of action executors for Java (e.g.
map-reduce, pig, hive, java, etc) -->
+
+ <property>
+ <name>oozie.action.launcher.mapreduce.job.ubertask.enable</name>
+ <value>true</value>
+ <description>
+ Enables Uber Mode for the launcher job in YARN/Hadoop 2 (no effect
in Hadoop 1).
+ Setting oozie.launcher.mapreduce.job.ubertask.enable in a an
action's configuration section overrides this for that
+ action.
+ </description>
+ </property>
<!-- HadoopActionExecutor -->
<!-- This is common to the subclasses action executors for map-reduce and
pig -->
Modified:
oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java?rev=1478460&r1=1478459&r2=1478460&view=diff
==============================================================================
---
oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
(original)
+++
oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
Thu May 2 17:34:39 2013
@@ -1440,4 +1440,50 @@ public class TestJavaActionExecutor exte
assertEquals("v3b", conf.get("p3"));
assertEquals("v4", conf.get("p4"));
}
+
+ public void testInjectLauncherUseUberMode() throws Exception {
+ // default -- should set to true
+ JavaActionExecutor jae = new JavaActionExecutor();
+ Configuration conf = new Configuration(false);
+ assertNull(conf.get("mapreduce.job.ubertask.enable"));
+ jae.injectLauncherUseUberMode(conf);
+ assertEquals("true", conf.get("mapreduce.job.ubertask.enable"));
+
+ // action conf set to true -- should keep at true
+ conf = new Configuration(false);
+ assertNull(conf.get("mapreduce.job.ubertask.enable"));
+ conf.setBoolean("mapreduce.job.ubertask.enable", true);
+ jae.injectLauncherUseUberMode(conf);
+ assertEquals("true", conf.get("mapreduce.job.ubertask.enable"));
+
+ // action conf set to false -- should keep at false
+ conf = new Configuration(false);
+ assertNull(conf.get("mapreduce.job.ubertask.enable"));
+ conf.setBoolean("mapreduce.job.ubertask.enable", false);
+ jae.injectLauncherUseUberMode(conf);
+ assertEquals("false", conf.get("mapreduce.job.ubertask.enable"));
+
+ // disable at oozie-site level (default is to be enabled) -- redo
above tests
+
Services.get().getConf().setBoolean("oozie.action.launcher.mapreduce.job.ubertask.enable",
false);
+
+ // default -- should not set
+ conf = new Configuration(false);
+ assertNull(conf.get("mapreduce.job.ubertask.enable"));
+ jae.injectLauncherUseUberMode(conf);
+ assertNull(conf.get("mapreduce.job.ubertask.enable"));
+
+ // action conf set to true -- should keep at true
+ conf = new Configuration(false);
+ assertNull(conf.get("mapreduce.job.ubertask.enable"));
+ conf.setBoolean("mapreduce.job.ubertask.enable", true);
+ jae.injectLauncherUseUberMode(conf);
+ assertEquals("true", conf.get("mapreduce.job.ubertask.enable"));
+
+ // action conf set to false -- should keep at false
+ conf = new Configuration(false);
+ assertNull(conf.get("mapreduce.job.ubertask.enable"));
+ conf.setBoolean("mapreduce.job.ubertask.enable", false);
+ jae.injectLauncherUseUberMode(conf);
+ assertEquals("false", conf.get("mapreduce.job.ubertask.enable"));
+ }
}
Modified: oozie/trunk/release-log.txt
URL:
http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1478460&r1=1478459&r2=1478460&view=diff
==============================================================================
--- oozie/trunk/release-log.txt (original)
+++ oozie/trunk/release-log.txt Thu May 2 17:34:39 2013
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1335 The launcher job should use uber mode in Hadoop 2 by default
(rkanter)
OOZIE-1297 Add chgrp in FS action (ryota via virag)
OOZIE-1329 fix coverage org.apache.oozie.tools (agorshkov via virag)
OOZIE-1351 Oozie jobs with state PAUSEDWITHERROR should change to
SUSPENDEDWITHERROR state when suspended (bowenzhangusa via virag)
