Repository: oozie Updated Branches: refs/heads/master bec42a030 -> 16fa65946
OOZIE-2512 ShareLibservice returns incorrect path for jar Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/16fa6594 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/16fa6594 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/16fa6594 Branch: refs/heads/master Commit: 16fa6594689d86dc4f4a62124f2b1140f0b9368d Parents: bec42a0 Author: Purshotam Shah <[email protected]> Authored: Thu Aug 11 12:44:44 2016 -0700 Committer: Purshotam Shah <[email protected]> Committed: Thu Aug 11 12:44:44 2016 -0700 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 4 +-- .../apache/oozie/service/ShareLibService.java | 10 +++---- .../oozie/service/TestShareLibService.java | 29 +++++++++++++++++++- release-log.txt | 1 + 4 files changed, 35 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/16fa6594/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 9e1682c..e546e77 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -683,12 +683,10 @@ public class JavaActionExecutor extends ActionExecutor { if (listOfPaths != null && !listOfPaths.isEmpty()) { for (Path actionLibPath : listOfPaths) { String fragmentName = new URI(actionLibPath.toString()).getFragment(); - Path pathWithFragment = fragmentName == null ? actionLibPath : new Path(new URI( - actionLibPath.toString()).getPath()); String fileName = fragmentName == null ? actionLibPath.getName() : fragmentName; if (confSet.contains(fileName)) { Configuration jobXmlConf = shareLibService.getShareLibConf(actionShareLibName, - pathWithFragment); + actionLibPath); if (jobXmlConf != null) { checkForDisallowedProps(jobXmlConf, actionLibPath.getName()); XConfiguration.injectDefaults(jobXmlConf, conf); http://git-wip-us.apache.org/repos/asf/oozie/blob/16fa6594/core/src/main/java/org/apache/oozie/service/ShareLibService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/ShareLibService.java b/core/src/main/java/org/apache/oozie/service/ShareLibService.java index 66fd864..b79bd37 100644 --- a/core/src/main/java/org/apache/oozie/service/ShareLibService.java +++ b/core/src/main/java/org/apache/oozie/service/ShareLibService.java @@ -289,7 +289,7 @@ public class ShareLibService implements Service, Instrumentable { Path filePath = new Path(new URI(rootDir.toString()).getPath()); Path qualifiedRootDirPath = fs.makeQualified(rootDir); if (isFilePartOfConfList(rootDir)) { - cachePropertyFile(filePath, shareLibKey, shareLibConfigMap); + cachePropertyFile(qualifiedRootDirPath, filePath, shareLibKey, shareLibConfigMap); } listOfPaths.add(qualifiedRootDirPath); return; @@ -307,7 +307,7 @@ public class ShareLibService implements Service, Instrumentable { } else { if (isFilePartOfConfList(file.getPath())) { - cachePropertyFile(file.getPath(), shareLibKey, shareLibConfigMap); + cachePropertyFile(file.getPath(), file.getPath(), shareLibKey, shareLibConfigMap); } listOfPaths.add(file.getPath()); } @@ -642,7 +642,7 @@ public class ShareLibService implements Service, Instrumentable { Path path = new Path(dfsPath); getPathRecursively(fs, new Path(dfsPath), listOfPaths, shareLibKey, shareLibConfigMap); if (HadoopShims.isSymlinkSupported() && fileSystem.isSymlink(path)) { - symlinkMappingforAction.put(path, fileSystem.getSymLinkTarget(path)); + symlinkMappingforAction.put(fs.makeQualified(path), fileSystem.getSymLinkTarget(path)); } } if (HadoopShims.isSymlinkSupported()) { @@ -834,7 +834,7 @@ public class ShareLibService implements Service, Instrumentable { * @throws IOException Signals that an I/O exception has occurred. * @throws JDOMException */ - private void cachePropertyFile(Path hdfsPath, String shareLibKey, + private void cachePropertyFile(Path qualifiedHdfsPath, Path hdfsPath, String shareLibKey, Map<String, Map<Path, Configuration>> shareLibConfigMap) throws IOException, JDOMException { Map<Path, Configuration> confMap = shareLibConfigMap.get(shareLibKey); if (confMap == null) { @@ -842,7 +842,7 @@ public class ShareLibService implements Service, Instrumentable { shareLibConfigMap.put(shareLibKey, confMap); } Configuration xmlConf = new XConfiguration(fs.open(hdfsPath)); - confMap.put(hdfsPath, xmlConf); + confMap.put(qualifiedHdfsPath, xmlConf); } http://git-wip-us.apache.org/repos/asf/oozie/blob/16fa6594/core/src/test/java/org/apache/oozie/service/TestShareLibService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java index 7de7d78..2d31f9a 100644 --- a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java +++ b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java @@ -902,7 +902,34 @@ public class TestShareLibService extends XFsTestCase { actionConf = ae.createBaseHadoopConf(context, eActionXml); prop.put("oozie.hive_conf", "hdfs:///user/test/" + sharelibPath + "/hive-site.xml#hive-site.xml"); setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop); - jobConf.set("oozie.action.sharelib.for.hive", "hive_conf,linkFile"); + jobConf.set("oozie.action.sharelib.for.hive", "hive_conf"); + ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf); + assertEquals(jobConf.get("oozie.hive_conf-sharelib-test"), "test"); + cacheFiles = DistributedCache.getCacheFiles(actionConf); + cacheFilesStr = Arrays.toString(cacheFiles); + assertFalse(cacheFilesStr.contains("hive-site.xml")); + + // Test hive-site.xml property in jobconf with non hdfs path + prop = new Properties(); + jobConf = ae.createBaseHadoopConf(context, eActionXml); + actionConf = ae.createBaseHadoopConf(context, eActionXml); + prop.put("oozie.hive_conf", "/user/test/" + sharelibPath + "/hive-site.xml"); + setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop); + jobConf.set("oozie.action.sharelib.for.hive", "hive_conf"); + ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf); + assertEquals(jobConf.get("oozie.hive_conf-sharelib-test"), "test"); + cacheFiles = DistributedCache.getCacheFiles(actionConf); + cacheFilesStr = Arrays.toString(cacheFiles); + assertFalse(cacheFilesStr.contains("hive-site.xml")); + + // Test hive-site.xml property in jobconf with non hdfs path with + // link name + prop = new Properties(); + jobConf = ae.createBaseHadoopConf(context, eActionXml); + actionConf = ae.createBaseHadoopConf(context, eActionXml); + prop.put("oozie.hive_conf", "/user/test/" + sharelibPath + "/hive-site.xml#hive-site.xml"); + setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop); + jobConf.set("oozie.action.sharelib.for.hive", "hive_conf"); ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf); assertEquals(jobConf.get("oozie.hive_conf-sharelib-test"), "test"); cacheFiles = DistributedCache.getCacheFiles(actionConf); http://git-wip-us.apache.org/repos/asf/oozie/blob/16fa6594/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 1531194..86a68c5 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2512 ShareLibservice returns incorrect path for jar (satishsaley via puru) OOZIE-2508 Documentation change for Coord action rerun [OOZIE-1735] (satishsaley via puru) OOZIE-2628 Fix Workflow EL function return types in Documentation (abhishekbafna via jaydeepvishwakarma) OOZIE-2623 Oozie should use a dummy OutputFormat (satishsaley via rohini)
