Author: ryota
Date: Tue Nov 19 00:55:09 2013
New Revision: 1543280
URL: http://svn.apache.org/r1543280
Log:
OOZIE-1604 <java-opts> and <java-opt> not added to Application Master property
in uber mode (ryota)
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
oozie/trunk/release-log.txt
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java?rev=1543280&r1=1543279&r2=1543280&view=diff
==============================================================================
---
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
(original)
+++
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
Tue Nov 19 00:55:09 2013
@@ -233,7 +233,6 @@ public class JavaActionExecutor extends
XConfiguration actionDefaultConf =
has.createActionDefaultConf(conf.get(HADOOP_JOB_TRACKER), getType());
injectLauncherProperties(actionDefaultConf, launcherConf);
injectLauncherProperties(inlineConf, launcherConf);
- injectLauncherUseUberMode(launcherConf);
checkForDisallowedProps(launcherConf, "launcher
configuration");
XConfiguration.copy(launcherConf, conf);
}
@@ -734,6 +733,9 @@ public class JavaActionExecutor extends
launcherJobConf.set("mapred.child.java.opts", opts);
}
+ // setting for uber mode
+ injectLauncherUseUberMode(launcherJobConf);
+
// properties from action that are needed by the launcher (e.g.
QUEUE NAME, ACLs)
// maybe we should add queue to the WF schema, below job-tracker
actionConfToLauncherConf(actionConf, launcherJobConf);
Modified:
oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java?rev=1543280&r1=1543279&r2=1543280&view=diff
==============================================================================
---
oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
(original)
+++
oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
Tue Nov 19 00:55:09 2013
@@ -1575,8 +1575,8 @@ public class TestJavaActionExecutor exte
action.setType(ae.getType());
Context context = new Context(wf, action);
- JobConf launcherConf = ae.createBaseHadoopConf(context, actionXml1);
- ae.setupLauncherConf(launcherConf, actionXml1, getFsTestCaseDir(),
context);
+ JobConf launcherConf = new JobConf();
+ launcherConf = ae.createLauncherConf(getFileSystem(), context, action,
actionXml1, launcherConf);
// memoryMB (2048 + 512)
assertEquals("2560",
launcherConf.get(JavaActionExecutor.YARN_AM_RESOURCE_MB));
// heap size in child.opts (2048 + 512)
@@ -1607,8 +1607,7 @@ public class TestJavaActionExecutor exte
+
"<property><name>oozie.launcher.mapred.child.env</name><value>B=bar</value></property>"
+ "</configuration>" +
"<main-class>MAIN-CLASS</main-class>" + "</java>");
- launcherConf = ae.createBaseHadoopConf(context, actionXml2);
- ae.setupLauncherConf(launcherConf, actionXml2, getFsTestCaseDir(),
context);
+ launcherConf = ae.createLauncherConf(getFileSystem(), context, action,
actionXml2, launcherConf);
// memoryMB (3072 + 512)
assertEquals("3584",
launcherConf.get(JavaActionExecutor.YARN_AM_RESOURCE_MB));
@@ -1644,7 +1643,7 @@ public class TestJavaActionExecutor exte
+ "</configuration>" +
"<main-class>MAIN-CLASS</main-class>" + "</java>");
launcherConf = ae.createBaseHadoopConf(context, actionXml3);
- ae.setupLauncherConf(launcherConf, actionXml3, getFsTestCaseDir(),
context);
+ launcherConf = ae.createLauncherConf(getFileSystem(), context, action,
actionXml3, launcherConf);
// memoryMB (limit to 4096)
assertEquals("4096",
launcherConf.get(JavaActionExecutor.YARN_AM_RESOURCE_MB));
@@ -1658,6 +1657,93 @@ public class TestJavaActionExecutor exte
assertEquals("A=foo,B=bar",
launcherConf.get(JavaActionExecutor.YARN_AM_ENV));
}
+ public void testUpdateConfForUberModeForJavaOpts() throws Exception {
+
Services.get().getConf().setBoolean("oozie.action.launcher.mapreduce.job.ubertask.enable",
true);
+
+ Element actionXml1 = XmlUtils
+ .parseXml("<java>"
+ + "<job-tracker>"
+ + getJobTrackerUri()
+ + "</job-tracker>"
+ + "<name-node>"
+ + getNameNodeUri()
+ + "</name-node>"
+ + "<configuration>"
+ +
"<property><name>oozie.launcher.yarn.app.mapreduce.am.command-opts</name>"
+ + "<value>-Xmx1024m -Djava.net.preferIPv4Stack=true
</value></property>"
+ +
"<property><name>oozie.launcher.mapred.child.java.opts</name><value>-Xmx1536m</value></property>"
+ + "</configuration>" +
"<main-class>MAIN-CLASS</main-class>"
+ + "<java-opt>-Xmx2048m</java-opt>"
+ + "<java-opt>-Dkey1=val1</java-opt>"
+ + "<java-opt>-Dkey2=val2</java-opt>"
+ + "</java>");
+ JavaActionExecutor ae = new JavaActionExecutor();
+ XConfiguration protoConf = new XConfiguration();
+ protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+
+ WorkflowJobBean wf = createBaseWorkflow(protoConf, "action");
+ WorkflowActionBean action = (WorkflowActionBean)
wf.getActions().get(0);
+ action.setType(ae.getType());
+
+ Context context = new Context(wf, action);
+ JobConf launcherConf = new JobConf();
+ launcherConf = ae.createLauncherConf(getFileSystem(), context, action,
actionXml1, launcherConf);
+
+ // heap size (2048 + 512)
+ int heapSize =
ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
+ assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx1536m
-Xmx2048m -Dkey1=val1 -Dkey2=val2 -Xmx2560m",
+
launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim());
+ assertEquals(2560, heapSize);
+
+ Element actionXml2 = XmlUtils
+ .parseXml("<java>"
+ + "<job-tracker>"
+ + getJobTrackerUri()
+ + "</job-tracker>"
+ + "<name-node>"
+ + getNameNodeUri()
+ + "</name-node>"
+ + "<configuration>"
+ +
"<property><name>oozie.launcher.yarn.app.mapreduce.am.command-opts</name>"
+ + "<value>-Xmx1024m -Djava.net.preferIPv4Stack=true
</value></property>"
+ +
"<property><name>oozie.launcher.mapred.child.java.opts</name><value>-Xmx1536m</value></property>"
+ + "</configuration>" +
"<main-class>MAIN-CLASS</main-class>"
+ + "<java-opts>-Xmx2048m -Dkey1=val1</java-opts>"
+ + "</java>");
+
+ launcherConf = ae.createLauncherConf(getFileSystem(), context, action,
actionXml2, launcherConf);
+
+ // heap size (2048 + 512)
+ heapSize =
ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
+ assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx1536m
-Xmx2048m -Dkey1=val1 -Xmx2560m",
+
launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim());
+ assertEquals(2560, heapSize);
+
+ Element actionXml3 = XmlUtils
+ .parseXml("<java>"
+ + "<job-tracker>"
+ + getJobTrackerUri()
+ + "</job-tracker>"
+ + "<name-node>"
+ + getNameNodeUri()
+ + "</name-node>"
+ + "<configuration>"
+ +
"<property><name>oozie.launcher.yarn.app.mapreduce.am.command-opts</name>"
+ + "<value>-Xmx2048m -Djava.net.preferIPv4Stack=true
</value></property>"
+ +
"<property><name>oozie.launcher.mapred.child.java.opts</name><value>-Xmx3072m</value></property>"
+ + "</configuration>" +
"<main-class>MAIN-CLASS</main-class>"
+ + "<java-opts>-Xmx1024m -Dkey1=val1</java-opts>"
+ + "</java>");
+
+ launcherConf = ae.createLauncherConf(getFileSystem(), context, action,
actionXml3, launcherConf);
+
+ // heap size (2048 + 512)
+ heapSize =
ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
+ assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true -Xmx3072m
-Xmx1024m -Dkey1=val1 -Xmx2560m",
+
launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim());
+ assertEquals(2560, heapSize);
+ }
+
public void testAddToCache() throws Exception {
JavaActionExecutor ae = new JavaActionExecutor();
Configuration conf = new XConfiguration();
Modified: oozie/trunk/release-log.txt
URL:
http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1543280&r1=1543279&r2=1543280&view=diff
==============================================================================
--- oozie/trunk/release-log.txt (original)
+++ oozie/trunk/release-log.txt Tue Nov 19 00:55:09 2013
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1604 <java-opts> and <java-opt> not added to Application Master property
in uber mode (ryota)
OOZIE-1584 Setup sharelib using script and pickup latest(honor ship.launcher)
and remove DFS dependency at startup (puru via ryota)
OOZIE-1550 Create a safeguard to kill errant recursive workflows before they
bring down oozie (rkanter)
OOZIE-1314 IllegalArgumentException: wfId cannot be empty (shwethags via virag)
