Repository: oozie Updated Branches: refs/heads/master 9c91c814c -> 6e2e46a98
OOZIE-1806 Java Action type jobs are failing with hadoop-0.20.0 and earlier versions on oozie trunk (sathish.mittal via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/6e2e46a9 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/6e2e46a9 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/6e2e46a9 Branch: refs/heads/master Commit: 6e2e46a98fa59377134ec0f5303ead1b6f1c7532 Parents: 9c91c81 Author: Rohini Palaniswamy <[email protected]> Authored: Tue May 6 10:16:45 2014 -0700 Committer: Rohini Palaniswamy <[email protected]> Committed: Tue May 6 10:16:45 2014 -0700 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 7 +++-- .../java/org/apache/oozie/util/JobUtils.java | 30 ++++++++++++++++++++ release-log.txt | 1 + 3 files changed, 35 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/6e2e46a9/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index d4b4f5e..40add2c 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -66,6 +66,7 @@ import org.apache.oozie.service.URIHandlerService; import org.apache.oozie.service.WorkflowAppService; import org.apache.oozie.servlet.CallbackServlet; import org.apache.oozie.util.ELEvaluator; +import org.apache.oozie.util.JobUtils; import org.apache.oozie.util.LogUtils; import org.apache.oozie.util.PropertiesUtils; import org.apache.oozie.util.XConfiguration; @@ -524,7 +525,7 @@ public class JavaActionExecutor extends ActionExecutor { if (listOfPaths != null && !listOfPaths.isEmpty()) { for (Path actionLibPath : listOfPaths) { - DistributedCache.addFileToClassPath(actionLibPath, conf, fs); + JobUtils.addFileToClassPath(actionLibPath, conf, fs); DistributedCache.createSymlink(conf); } } @@ -554,13 +555,13 @@ public class JavaActionExecutor extends ActionExecutor { } FileSystem fs = listOfPaths.get(0).getFileSystem(conf); for (Path actionLibPath : listOfPaths) { - DistributedCache.addFileToClassPath(actionLibPath, conf, fs); + JobUtils.addFileToClassPath(actionLibPath, conf, fs); DistributedCache.createSymlink(conf); } listOfPaths = shareLibService.getSystemLibJars(getType()); if (listOfPaths != null) { for (Path actionLibPath : listOfPaths) { - DistributedCache.addFileToClassPath(actionLibPath, conf, fs); + JobUtils.addFileToClassPath(actionLibPath, conf, fs); DistributedCache.createSymlink(conf); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/6e2e46a9/core/src/main/java/org/apache/oozie/util/JobUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/JobUtils.java b/core/src/main/java/org/apache/oozie/util/JobUtils.java index 25eb735..135b096 100644 --- a/core/src/main/java/org/apache/oozie/util/JobUtils.java +++ b/core/src/main/java/org/apache/oozie/util/JobUtils.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.oozie.ErrorCode; @@ -133,4 +134,33 @@ public class JobUtils { return map; } + + /** + * This method provides a wrapper around hadoop 0.20/1.x and 0.23/2.x implementations. + * TODO: Remove the workaround when we drop the support for hadoop 0.20. + * @param file Path of the file to be added + * @param conf Configuration that contains the classpath setting + * @param fs FileSystem with respect to which path should be interpreted + * @throws IOException + */ + public static void addFileToClassPath(Path file, Configuration conf, FileSystem fs) throws IOException { + Configuration defaultConf = new Configuration(); + XConfiguration.copy(conf, defaultConf); + DistributedCache.addFileToClassPath(file, defaultConf, fs); + // Hadoop 0.20/1.x. + if (defaultConf.get("mapred.job.classpath.files") != null) { + // Duplicate hadoop 1.x code to workaround MAPREDUCE-2361 in Hadoop 0.20 + // Refer OOZIE-1806. + String filepath = file.toUri().getPath(); + String classpath = conf.get("mapred.job.classpath.files"); + conf.set("mapred.job.classpath.files", classpath == null + ? filepath + : classpath + System.getProperty("path.separator") + filepath); + URI uri = fs.makeQualified(file).toUri(); + DistributedCache.addCacheFile(uri, conf); + } + else { // Hadoop 0.23/2.x + DistributedCache.addFileToClassPath(file, conf, fs); + } + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/6e2e46a9/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index b2fcf6d..795311a 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.1.0 release (trunk - unreleased) +OOZIE-1806 Java Action type jobs are failing with hadoop-0.20.0 and earlier versions on oozie trunk (sathish.mittal via rohini) OOZIE-1823 OozieSharelibCLI shouldn't load ext services (rkanter) OOZIE-1762 Sharelib with oozie.action.ship.launcher.jar=true should copy oozie-hadoop-utils.jar (puru via mona) OOZIE-1709 CoordELFunctions.getCurrentInstance() is expensive (shwethags via rohini)
