Updated Branches: refs/heads/master 1eaf7b9eb -> ab67228ce
OOZIE-1675 Adding absolute URI of local cluster to dist cache not working with hadoop version 0.20.2 and before (satish via ryota) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/ab67228c Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/ab67228c Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/ab67228c Branch: refs/heads/master Commit: ab67228ce31afac65f668579507f6e1c17a8301e Parents: 1eaf7b9 Author: egashira <[email protected]> Authored: Sat Feb 1 01:10:51 2014 -0800 Committer: egashira <[email protected]> Committed: Sat Feb 1 01:10:51 2014 -0800 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 14 ++++++++++- .../action/hadoop/TestJavaActionExecutor.java | 26 +++++++++++++++++++- release-log.txt | 1 + 3 files changed, 39 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/ab67228c/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 5cd9eb0..68d77a8 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -426,7 +426,19 @@ public class JavaActionExecutor extends ActionExecutor { else if (fileName.endsWith(".jar")) { // .jar files if (!fileName.contains("#")) { String user = conf.get("user.name"); - Services.get().get(HadoopAccessorService.class).addFileToClassPath(user, new Path(uri.normalize()), conf); + Path pathToAdd; + // if filePath and appPath belong to same cluster, add URI path component else add absolute URI + if (uri.getScheme() != null && uri.getHost() != null && + uri.getPort() > 0 && baseUri.getScheme() != null && + baseUri.getHost() != null && baseUri.getPort() > 0 && + uri.getScheme().equalsIgnoreCase(baseUri.getScheme()) && + uri.getHost().equalsIgnoreCase(baseUri.getHost()) && + uri.getPort() == baseUri.getPort()) { + pathToAdd = new Path(uri.getPath()); + } else { + pathToAdd = new Path(uri.normalize()); + } + Services.get().get(HadoopAccessorService.class).addFileToClassPath(user, pathToAdd, conf); } else { DistributedCache.addCacheFile(uri, conf); http://git-wip-us.apache.org/repos/asf/oozie/blob/ab67228c/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index 88af9b5..7841076 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -1796,12 +1796,36 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertTrue(conf.get("mapred.cache.files").contains(appSoFragmentFullPath.toString())); assertTrue(DistributedCache.getSymlink(conf)); - // test .jar without fragment + // test .jar without fragment where app path is on same cluster as jar path Path appJarPath = new Path("lib/a.jar"); Path appJarFullPath = new Path(appPath, appJarPath); conf.clear(); conf.set(WorkflowAppService.HADOOP_USER, getTestUser()); ae.addToCache(conf, appPath, appJarFullPath.toString(), false); + // assert that mapred.cache.files contains jar URI path + Path jarPath = new Path(appJarFullPath.toUri().getPath()); + assertTrue(conf.get("mapred.cache.files").contains(jarPath.toString())); + // assert that dist cache classpath contains jar URI path + Path[] paths = DistributedCache.getFileClassPaths(conf); + boolean pathFound = false; + for (Path path : paths) { + if (path.equals(jarPath)) { + pathFound = true; + break; + } + } + assertTrue(pathFound); + assertTrue(DistributedCache.getSymlink(conf)); + + // test .jar without fragment where app path is on a different cluster than jar path + appJarPath = new Path("lib/a.jar"); + appJarFullPath = new Path(appPath, appJarPath); + Path appDifferentClusterPath = new Path(new URI(appUri.getScheme(), null, appUri.getHost() + "x", + appUri.getPort(), appUri.getPath(), appUri.getQuery(), appUri.getFragment())); + conf.clear(); + conf.set(WorkflowAppService.HADOOP_USER, getTestUser()); + ae.addToCache(conf, appDifferentClusterPath, appJarFullPath.toString(), false); + // assert that mapred.cache.files contains absolute jar URI assertTrue(conf.get("mapred.cache.files").contains(appJarFullPath.toString())); assertTrue(DistributedCache.getSymlink(conf)); http://git-wip-us.apache.org/repos/asf/oozie/blob/ab67228c/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 2b70dfc..f424dd3 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.1.0 release (trunk - unreleased) +OOZIE-1675 Adding absolute URI of local cluster to dist cache not working with hadoop version 0.20.2 and before (satish via ryota) OOZIE-1683 UserGroupInformationService should close any filesystems opened by it (rkanter) OOZIE-1646 HBase Table Copy between two HBase servers doesn't work with Kerberos (rkanter) OOZIE-1609 HA support for share lib. (puru via rkanter)
