Repository: oozie Updated Branches: refs/heads/master d425d83e1 -> 168de6607
Amending OOZIE-2260 to fix test case failures against hadoop-2 (ryota) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/168de660 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/168de660 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/168de660 Branch: refs/heads/master Commit: 168de660764719547bd3085e4ee1d3d7c85a1cbc Parents: d425d83 Author: egashira <[email protected]> Authored: Tue Jun 23 15:59:40 2015 -0700 Committer: egashira <[email protected]> Committed: Tue Jun 23 16:00:06 2015 -0700 ---------------------------------------------------------------------- .../action/hadoop/TestJavaActionExecutor.java | 31 ++++++++++++++------ .../apache/oozie/hadoop/utils/HadoopShims.java | 2 +- .../apache/oozie/hadoop/utils/HadoopShims.java | 2 +- .../apache/oozie/hadoop/utils/HadoopShims.java | 2 +- 4 files changed, 25 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/168de660/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index 1a10352..21e85d1 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -50,6 +50,7 @@ import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.OozieClient; import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.client.WorkflowJob; +import org.apache.oozie.hadoop.utils.HadoopShims; import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.HadoopAccessorService; import org.apache.oozie.service.LiteWorkflowStoreService; @@ -1781,7 +1782,12 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals("-Xmx200m", jobConf.get(JavaActionExecutor.HADOOP_CHILD_JAVA_OPTS)); assertEquals("-Xmx1024m -Djava.io.tmpdir=./usr", jobConf.get(JavaActionExecutor.HADOOP_MAP_JAVA_OPTS)); assertEquals("-Xmx2560m -XX:NewRatio=8", jobConf.get(JavaActionExecutor.HADOOP_REDUCE_JAVA_OPTS)); - assertNull(jobConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS)); + // There's an extra parameter (-Xmx1024m) in here when using YARN that's not here when using MR1 + if (HadoopShims.isYARN()) { + assertEquals("-Xmx1024m -Djava.io.tmpdir=./tmp", jobConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS)); + } else { + assertNull(jobConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS)); + } } public void testUpdateConfForUberMode() throws Exception { Element actionXml1 = XmlUtils @@ -1818,8 +1824,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true", launcherConf.get("mapreduce.map.java.opts")); // There's an extra parameter (-Xmx1024m) in here when using YARN that's not here when using MR1 - if (createJobConf().get("yarn.resourcemanager.address") != null) { - assertEquals("-Xmx1024m -Xmx2048m -Djava.net.preferIPv4Stack=true -Xmx2560m", + if (HadoopShims.isYARN()) { + assertEquals("-Xmx1024m -Xmx2048m -Djava.net.preferIPv4Stack=true -Xmx2560m -Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim()); } else { @@ -2137,14 +2143,21 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { // Test when server side setting is not enabled JobConf launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, actionConf); - assertNull(launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED)); + if (HadoopShims.isYARN()) { + assertEquals("true", launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED)); + } else { + assertNull(launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED)); + } ConfigurationService.set("oozie.action.launcher." + JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED, "true"); // Test when server side setting is enabled but tez-site.xml is not in DistributedCache launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, actionConf); - assertNull(launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED)); - + if (HadoopShims.isYARN()) { + assertEquals("true", launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED)); + } else { + assertNull(launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED)); + } final Path tezSite = new Path("/tmp/tez-site.xml"); final FSDataOutputStream out = getFileSystem().create(tezSite); out.close(); @@ -2234,9 +2247,9 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { conf.set(WorkflowAppService.HADOOP_USER, getTestUser()); ae.addToCache(conf, appPath, appJarFullPath.toString(), false); // assert that mapred.cache.files contains jar URI path (full on Hadoop-2) - Path jarPath = createJobConf().get("yarn.resourcemanager.address") == null ? - new Path(appJarFullPath.toUri().getPath()) : - new Path(appJarFullPath.toUri()); + Path jarPath = HadoopShims.isYARN() ? + new Path(appJarFullPath.toUri()): + new Path(appJarFullPath.toUri().getPath()); assertTrue(conf.get("mapred.cache.files").contains(jarPath.toString())); // assert that dist cache classpath contains jar URI path Path[] paths = DistributedCache.getFileClassPaths(conf); http://git-wip-us.apache.org/repos/asf/oozie/blob/168de660/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java index 559ce8e..799dffb 100644 --- a/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java +++ b/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java @@ -44,7 +44,7 @@ public class HadoopShims { public void createSymlink(Path target, Path link, boolean createParent) throws IOException { } - public boolean isYARN() { + public static boolean isYARN() { return false; } http://git-wip-us.apache.org/repos/asf/oozie/blob/168de660/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java index d7b4051..c21884c 100644 --- a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java +++ b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java @@ -61,7 +61,7 @@ public class HadoopShims { fs.createSymlink(target, link, createParent); } - public boolean isYARN() { + public static boolean isYARN() { return true; } http://git-wip-us.apache.org/repos/asf/oozie/blob/168de660/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java index d7b4051..c21884c 100644 --- a/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java +++ b/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java @@ -61,7 +61,7 @@ public class HadoopShims { fs.createSymlink(target, link, createParent); } - public boolean isYARN() { + public static boolean isYARN() { return true; }
