Repository: oozie Updated Branches: refs/heads/master 48b64bc94 -> 999fbcdcb
OOZIE-2260 Only set java.io.tmpdir to ./tmp for the AM (ryota) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/999fbcdc Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/999fbcdc Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/999fbcdc Branch: refs/heads/master Commit: 999fbcdcb225d0c7b3129c8bf54b3b123f180e1c Parents: 48b64bc Author: egashira <[email protected]> Authored: Tue Jun 9 02:10:42 2015 -0700 Committer: egashira <[email protected]> Committed: Tue Jun 9 02:10:42 2015 -0700 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 17 ------- .../action/hadoop/TestJavaActionExecutor.java | 52 ++++++++++---------- release-log.txt | 1 + 3 files changed, 27 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/999fbcdc/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 695853e..7a7d4a7 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -407,25 +407,8 @@ public class JavaActionExecutor extends ActionExecutor { } void updateConfForJavaTmpDir(Configuration conf) { - String mapOpts = conf.get(HADOOP_MAP_JAVA_OPTS); - String reduceOpts = conf.get(HADOOP_REDUCE_JAVA_OPTS); - String childOpts = conf.get(HADOOP_CHILD_JAVA_OPTS); String amChildOpts = conf.get(YARN_AM_COMMAND_OPTS); String oozieJavaTmpDirSetting = "-Djava.io.tmpdir=./tmp"; - if (childOpts == null) { - conf.set(HADOOP_CHILD_JAVA_OPTS, oozieJavaTmpDirSetting); - } else { - conf.set(HADOOP_CHILD_JAVA_OPTS, childOpts + " " + oozieJavaTmpDirSetting); - } - - if (mapOpts != null && !mapOpts.contains(JAVA_TMP_DIR_SETTINGS)) { - conf.set(HADOOP_MAP_JAVA_OPTS, mapOpts + " " + oozieJavaTmpDirSetting); - } - - if (reduceOpts != null && !reduceOpts.contains(JAVA_TMP_DIR_SETTINGS)) { - conf.set(HADOOP_REDUCE_JAVA_OPTS, reduceOpts + " " + oozieJavaTmpDirSetting); - } - if (amChildOpts != null && !amChildOpts.contains(JAVA_TMP_DIR_SETTINGS)) { conf.set(YARN_AM_COMMAND_OPTS, amChildOpts + " " + oozieJavaTmpDirSetting); } http://git-wip-us.apache.org/repos/asf/oozie/blob/999fbcdc/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index c34020b..1a10352 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -1070,8 +1070,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { Configuration conf = ae.createLauncherConf(getFileSystem(), context, action, actionXmlconf, actionConf); - assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapred.child.java.opts")); - assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapreduce.map.java.opts")); + assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2", conf.get("mapred.child.java.opts")); + assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2", conf.get("mapreduce.map.java.opts")); actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" @@ -1095,8 +1095,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { conf = ae.createLauncherConf(getFileSystem(), context, action, actionXmlconf, actionConf); - assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapred.child.java.opts")); - assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapreduce.map.java.opts")); + assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2", conf.get("mapred.child.java.opts")); + assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2", conf.get("mapreduce.map.java.opts")); actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" @@ -1122,8 +1122,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { conf = ae.createLauncherConf(getFileSystem(), context, action, actionXmlconf, actionConf); - assertEquals("JAVA-OPT3 JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapred.child.java.opts")); - assertEquals("JAVA-OPT3 JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapreduce.map.java.opts")); + assertEquals("JAVA-OPT3 JAVA-OPT1 JAVA-OPT2", conf.get("mapred.child.java.opts")); + assertEquals("JAVA-OPT3 JAVA-OPT1 JAVA-OPT2", conf.get("mapreduce.map.java.opts")); actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" @@ -1149,8 +1149,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { conf = ae.createLauncherConf(getFileSystem(), context, action, actionXmlconf, actionConf); - assertEquals("-Xmx200m JAVA-OPT3 JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapred.child.java.opts")); - assertEquals("-Xmx200m JAVA-OPT3 JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapreduce.map.java.opts")); + assertEquals("-Xmx200m JAVA-OPT3 JAVA-OPT1 JAVA-OPT2", conf.get("mapred.child.java.opts")); + assertEquals("-Xmx200m JAVA-OPT3 JAVA-OPT1 JAVA-OPT2", conf.get("mapreduce.map.java.opts")); actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" @@ -1177,8 +1177,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { conf = ae.createLauncherConf(getFileSystem(), context, action, actionXmlconf, actionConf); - assertEquals("JAVA-OPT3 JAVA-OPT4 JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapred.child.java.opts")); - assertEquals("JAVA-OPT3 JAVA-OPT4 JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapreduce.map.java.opts")); + assertEquals("JAVA-OPT3 JAVA-OPT4 JAVA-OPT1 JAVA-OPT2", conf.get("mapred.child.java.opts")); + assertEquals("JAVA-OPT3 JAVA-OPT4 JAVA-OPT1 JAVA-OPT2", conf.get("mapreduce.map.java.opts")); } public void testActionLibsPath() throws Exception { @@ -1749,9 +1749,9 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { Configuration actionConf = ae.createBaseHadoopConf(context, actionXmlconf); Configuration conf = ae.createLauncherConf(getFileSystem(), context, action, actionXmlconf, actionConf); - assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true -Djava.io.tmpdir=./tmp", + assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true", conf.get(JavaActionExecutor.HADOOP_CHILD_JAVA_OPTS)); - assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true -Djava.io.tmpdir=./tmp", + assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true", conf.get(JavaActionExecutor.HADOOP_MAP_JAVA_OPTS)); assertEquals("-Xmx2560m -XX:NewRatio=8 -Djava.io.tmpdir=./usr", conf.get(JavaActionExecutor.HADOOP_REDUCE_JAVA_OPTS)); assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Djava.io.tmpdir=./usr -Xmx2048m " + @@ -1778,9 +1778,9 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { Configuration jobConf = ae2.createBaseHadoopConf(context2, eActionXml); ae2.setupActionConf(jobConf, context2, eActionXml, appPath2); - assertEquals("-Xmx200m -Djava.io.tmpdir=./tmp", jobConf.get(JavaActionExecutor.HADOOP_CHILD_JAVA_OPTS)); + assertEquals("-Xmx200m", jobConf.get(JavaActionExecutor.HADOOP_CHILD_JAVA_OPTS)); assertEquals("-Xmx1024m -Djava.io.tmpdir=./usr", jobConf.get(JavaActionExecutor.HADOOP_MAP_JAVA_OPTS)); - assertEquals("-Xmx2560m -XX:NewRatio=8 -Djava.io.tmpdir=./tmp", jobConf.get(JavaActionExecutor.HADOOP_REDUCE_JAVA_OPTS)); + assertEquals("-Xmx2560m -XX:NewRatio=8", jobConf.get(JavaActionExecutor.HADOOP_REDUCE_JAVA_OPTS)); assertNull(jobConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS)); } public void testUpdateConfForUberMode() throws Exception { @@ -1813,9 +1813,9 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals("2560", launcherConf.get(JavaActionExecutor.YARN_AM_RESOURCE_MB)); // heap size in child.opts (2048 + 512) int heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS)); - assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true -Djava.io.tmpdir=./tmp", + assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true", launcherConf.get("mapred.child.java.opts")); - assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true -Djava.io.tmpdir=./tmp", + assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true", launcherConf.get("mapreduce.map.java.opts")); // There's an extra parameter (-Xmx1024m) in here when using YARN that's not here when using MR1 if (createJobConf().get("yarn.resourcemanager.address") != null) { @@ -1858,8 +1858,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { // heap size (2560 + 512) heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS)); - assertEquals("-Xmx1536m -Xmx2560m -XX:NewRatio=8 -Djava.io.tmpdir=./tmp", launcherConf.get("mapred.child.java.opts")); - assertEquals("-Xmx1536m -Xmx2560m -XX:NewRatio=8 -Djava.io.tmpdir=./tmp", launcherConf.get("mapreduce.map.java.opts")); + assertEquals("-Xmx1536m -Xmx2560m -XX:NewRatio=8", launcherConf.get("mapred.child.java.opts")); + assertEquals("-Xmx1536m -Xmx2560m -XX:NewRatio=8", launcherConf.get("mapreduce.map.java.opts")); assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx1536m -Xmx2560m -XX:NewRatio=8 " + "-Xmx3072m -Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim()); assertEquals(3072, heapSize); @@ -1896,8 +1896,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { // heap size (limit to 3584) heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS)); - assertEquals("-Xmx1536m -Xmx4000m -XX:NewRatio=8 -Djava.io.tmpdir=./tmp", launcherConf.get("mapred.child.java.opts")); - assertEquals("-Xmx1536m -Xmx4000m -XX:NewRatio=8 -Djava.io.tmpdir=./tmp", launcherConf.get("mapreduce.map.java.opts")); + assertEquals("-Xmx1536m -Xmx4000m -XX:NewRatio=8", launcherConf.get("mapred.child.java.opts")); + assertEquals("-Xmx1536m -Xmx4000m -XX:NewRatio=8", launcherConf.get("mapreduce.map.java.opts")); assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx1536m -Xmx4000m -XX:NewRatio=8 " + "-Xmx3584m -Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim()); assertEquals(3584, heapSize); @@ -2004,9 +2004,9 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { // heap size (2048 + 512) int heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS)); - assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Dkey2=val2 -Djava.io.tmpdir=./tmp", + assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Dkey2=val2", launcherConf.get("mapred.child.java.opts")); - assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Dkey2=val2 -Djava.io.tmpdir=./tmp", + assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Dkey2=val2", launcherConf.get("mapreduce.map.java.opts")); assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Dkey2=val2 -Xmx2560m " + "-Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim()); @@ -2032,9 +2032,9 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { // heap size (2048 + 512) heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS)); - assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Djava.io.tmpdir=./tmp", + assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1", launcherConf.get("mapred.child.java.opts")); - assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Djava.io.tmpdir=./tmp", + assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1", launcherConf.get("mapreduce.map.java.opts")); assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Xmx2560m " + "-Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim()); @@ -2060,9 +2060,9 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { // heap size (2048 + 512) heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS)); - assertEquals("-Xmx200m -Xmx3072m -Xmx1024m -Dkey1=val1 -Djava.io.tmpdir=./tmp", + assertEquals("-Xmx200m -Xmx3072m -Xmx1024m -Dkey1=val1", launcherConf.get("mapred.child.java.opts")); - assertEquals("-Xmx200m -Xmx3072m -Xmx1024m -Dkey1=val1 -Djava.io.tmpdir=./tmp", + assertEquals("-Xmx200m -Xmx3072m -Xmx1024m -Dkey1=val1", launcherConf.get("mapreduce.map.java.opts")); assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true -Xmx200m -Xmx3072m -Xmx1024m -Dkey1=val1 -Xmx2560m " + "-Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim()); http://git-wip-us.apache.org/repos/asf/oozie/blob/999fbcdc/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 809020a..78c6494 100644 --- a/release-log.txt +++ b/release-log.txt @@ -2,6 +2,7 @@ -- Oozie 4.2.0 release (unreleased) +OOZIE-2260 Only set "java.io.tmpdir" to "./tmp" for the AM (ryota) OOZIE-2240 add configuration to disable email attachment support (egashira via shwethags) OOZIE-1963 Create a Hive Server 2 example (qwertymaniac via shwethags) OOZIE-1993 Rerun fails during join in certain condition (shwethags)
