Repository: oozie Updated Branches: refs/heads/master a40535869 -> b5d409e63
OOZIE-2045 Symlink support for sharelib Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/b5d409e6 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/b5d409e6 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/b5d409e6 Branch: refs/heads/master Commit: b5d409e635dccef2dcf8f1137e754deb20f31461 Parents: a405358 Author: Purshotam Shah <[email protected]> Authored: Mon Nov 10 14:27:11 2014 -0800 Committer: Purshotam Shah <[email protected]> Committed: Mon Nov 10 14:27:11 2014 -0800 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 3 +- .../apache/oozie/service/ShareLibService.java | 155 ++++++++++++++----- .../oozie/service/TestShareLibService.java | 132 ++++++++++++---- .../apache/oozie/hadoop/utils/HadoopShims.java | 47 ++++++ .../apache/oozie/hadoop/utils/HadoopShims.java | 48 ++++++ .../apache/oozie/hadoop/utils/HadoopShims.java | 48 ++++++ .../apache/oozie/hadoop/utils/HadoopShims.java | 48 ++++++ release-log.txt | 1 + 8 files changed, 419 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/b5d409e6/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 687a948..fd1a98b 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -66,7 +66,6 @@ import org.apache.oozie.service.Services; import org.apache.oozie.service.ShareLibService; import org.apache.oozie.service.URIHandlerService; import org.apache.oozie.service.WorkflowAppService; -import org.apache.oozie.servlet.CallbackServlet; import org.apache.oozie.util.ELEvaluator; import org.apache.oozie.util.JobUtils; import org.apache.oozie.util.LogUtils; @@ -80,6 +79,7 @@ import org.jdom.JDOMException; import org.jdom.Namespace; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.oozie.hadoop.utils.HadoopShims; public class JavaActionExecutor extends ActionExecutor { @@ -138,6 +138,7 @@ public class JavaActionExecutor extends ActionExecutor { classes.add(LauncherMapper.class); classes.add(OozieLauncherInputFormat.class); classes.add(LauncherMainHadoopUtils.class); + classes.add(HadoopShims.class); classes.addAll(Services.get().get(URIHandlerService.class).getClassesForLauncher()); return classes; } http://git-wip-us.apache.org/repos/asf/oozie/blob/b5d409e6/core/src/main/java/org/apache/oozie/service/ShareLibService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/ShareLibService.java b/core/src/main/java/org/apache/oozie/service/ShareLibService.java index 5414e6b..b408549 100644 --- a/core/src/main/java/org/apache/oozie/service/ShareLibService.java +++ b/core/src/main/java/org/apache/oozie/service/ShareLibService.java @@ -21,8 +21,10 @@ package org.apache.oozie.service; import java.io.File; import java.io.IOException; import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.net.URLDecoder; +import java.text.MessageFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -38,6 +40,7 @@ import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.TimeZone; +import java.util.Map.Entry; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FileStatus; @@ -48,6 +51,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.oozie.action.ActionExecutor; import org.apache.oozie.action.hadoop.JavaActionExecutor; import org.apache.oozie.client.rest.JsonUtils; +import org.apache.oozie.hadoop.utils.HadoopShims; import org.apache.oozie.util.Instrumentable; import org.apache.oozie.util.Instrumentation; import org.apache.oozie.util.XLog; @@ -82,6 +86,9 @@ public class ShareLibService implements Service, Instrumentable { private Map<String, List<Path>> launcherLibMap = new HashMap<String, List<Path>>(); + //symlink mapping. Oozie keeps on checking symlink path and if changes, Oozie reloads the sharelib + private Map<String, Map<Path, Path>> symlinkMapping = new HashMap<String, Map<Path, Path>>(); + private static XLog LOG = XLog.getLog(ShareLibService.class); private String sharelibMappingFile; @@ -113,7 +120,8 @@ public class ShareLibService implements Service, Instrumentable { fs = FileSystem.get(has.createJobConf(uri.getAuthority())); updateLauncherLib(); updateShareLib(); - } catch(Throwable e) { + } + catch (Throwable e) { if (failOnfailure) { LOG.error("Sharelib initialization fails", e); throw new ServiceException(ErrorCode.E0104, getClass().getName(), "Sharelib initialization fails. ", e); @@ -132,7 +140,7 @@ public class ShareLibService implements Service, Instrumentable { public void run() { System.out.flush(); try { - //Only one server should purge sharelib + // Only one server should purge sharelib if (Services.get().get(JobsConcurrencyService.class).isLeader()) { final Date current = Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime(); purgeLibs(fs, LAUNCHER_PREFIX, current); @@ -228,7 +236,7 @@ public class ShareLibService implements Service, Instrumentable { * @param classes the classes * @param fs the FileSystem * @param executorDir is Path - * @param type is actionKey + * @param type is sharelib key * @throws IOException Signals that an I/O exception has occurred. */ private void copyJarContainingClasses(List<Class> classes, FileSystem fs, Path executorDir, String type) @@ -270,6 +278,15 @@ public class ShareLibService implements Service, Instrumentable { return; } + try { + if(fs.isFile(new Path(new URI(rootDir.toString()).getPath()))){ + listOfPaths.add(rootDir); + return; + } + } + catch (URISyntaxException e) { + throw new IOException(e); + } FileStatus[] status = fs.listStatus(rootDir); if (status == null) { LOG.info("Shared lib " + rootDir + " doesn't exist, not adding to cache"); @@ -290,14 +307,18 @@ public class ShareLibService implements Service, Instrumentable { return shareLibMap; } + private Map<String, Map<Path, Path>> getSymlinkMapping() { + return symlinkMapping; + } + /** - * Gets the action system lib common jars. + * Gets the action sharelib lib jars. * - * @param actionKey the action key + * @param shareLibKey the sharelib key * @return List of paths * @throws IOException */ - public List<Path> getShareLibJars(String actionKey) throws IOException { + public List<Path> getShareLibJars(String shareLibKey) throws IOException { // Sharelib map is empty means that on previous or startup attempt of // caching sharelib has failed.Trying to reload if (shareLibMap.isEmpty() && !shareLibLoadAttempted) { @@ -308,18 +329,45 @@ public class ShareLibService implements Service, Instrumentable { } } } - return shareLibMap.get(actionKey); + checkSymlink(shareLibKey); + return shareLibMap.get(shareLibKey); + } + + private void checkSymlink(String shareLibKey) throws IOException { + if (!HadoopShims.isSymlinkSupported() || symlinkMapping.get(shareLibKey) == null + || symlinkMapping.get(shareLibKey).isEmpty()) { + return; + } + + HadoopShims fileSystem = new HadoopShims(fs); + for (Path path : symlinkMapping.get(shareLibKey).keySet()) { + if (!symlinkMapping.get(shareLibKey).get(path).equals(fileSystem.getSymLinkTarget(path))) { + synchronized (ShareLibService.class) { + Map<String, List<Path>> tempShareLibMap = new HashMap<String, List<Path>>(shareLibMap); + Map<String, Map<Path, Path>> tmpSymlinkMapping = new HashMap<String, Map<Path, Path>>(symlinkMapping); + + LOG.info(MessageFormat.format("Symlink target for [{0}] has changed, was [{1}], now [{2}]", shareLibKey, + path, fileSystem.getSymLinkTarget(path))); + loadShareLibMetaFile(tempShareLibMap, tmpSymlinkMapping, sharelibMappingFile, shareLibKey); + shareLibMap = tempShareLibMap; + symlinkMapping = tmpSymlinkMapping; + return; + } + + } + } + } /** * Gets the launcher jars. * - * @param actionKey the action key + * @param shareLibKey the shareLib key * @return launcher jars paths * @throws ClassNotFoundException * @throws IOException */ - public List<Path> getSystemLibJars(String actionKey) throws IOException { + public List<Path> getSystemLibJars(String shareLibKey) throws IOException { List<Path> returnList = new ArrayList<Path>(); // Sharelib map is empty means that on previous or startup attempt of // caching launcher jars has failed.Trying to reload @@ -331,10 +379,10 @@ public class ShareLibService implements Service, Instrumentable { } } } - returnList.addAll(launcherLibMap.get(actionKey)); + returnList.addAll(launcherLibMap.get(shareLibKey)); } - if (actionKey.equals(JavaActionExecutor.OOZIE_COMMON_LIBDIR)) { - List<Path> sharelibList = getShareLibJars(actionKey); + if (shareLibKey.equals(JavaActionExecutor.OOZIE_COMMON_LIBDIR)) { + List<Path> sharelibList = getShareLibJars(shareLibKey); if (sharelibList != null) { returnList.addAll(sharelibList); } @@ -422,7 +470,7 @@ public class ShareLibService implements Service, Instrumentable { } }); - //Logic is to keep all share-lib between current timestamp and 7days old + 1 latest sharelib older than 7 days. + // Logic is to keep all share-lib between current timestamp and 7days old + 1 latest sharelib older than 7 days. // refer OOZIE-1761 for (int i = 1; i < dirList.length; i++) { Path dirPath = dirList[i].getPath(); @@ -460,11 +508,12 @@ public class ShareLibService implements Service, Instrumentable { } Map<String, List<Path>> tempShareLibMap = new HashMap<String, List<Path>>(); + Map<String, Map<Path, Path>> tmpSymlinkMapping = new HashMap<String, Map<Path, Path>>(); if (!StringUtils.isEmpty(sharelibMappingFile.trim())) { - String sharelibMetaFileNewTimeStamp = JsonUtils.formatDateRfc822(new Date(fs.getFileStatus( - new Path(sharelibMappingFile)).getModificationTime()),"GMT"); - loadShareLibMetaFile(tempShareLibMap, sharelibMappingFile); + String sharelibMetaFileNewTimeStamp = JsonUtils.formatDateRfc822( + new Date(fs.getFileStatus(new Path(sharelibMappingFile)).getModificationTime()), "GMT"); + loadShareLibMetaFile(tempShareLibMap, tmpSymlinkMapping, sharelibMappingFile, null); status.put("sharelibMetaFile", sharelibMappingFile); status.put("sharelibMetaFileNewTimeStamp", sharelibMetaFileNewTimeStamp); status.put("sharelibMetaFileOldTimeStamp", sharelibMetaFileOldTimeStamp); @@ -483,12 +532,12 @@ public class ShareLibService implements Service, Instrumentable { } shareLibMap = tempShareLibMap; + symlinkMapping = tmpSymlinkMapping; return status; } /** - * Update share lib cache. Parse the share lib directory and each sub - * directory is a action key + * Update share lib cache. Parse the share lib directory and each sub directory is a action key * * @param shareLibMap the share lib jar map * @param shareLibpath the share libpath @@ -522,16 +571,17 @@ public class ShareLibService implements Service, Instrumentable { } /** - * Load share lib text file. Sharelib mapping files contains list of - * key=value. where key is the action key and value is the DFS location of - * sharelib files. + * Load share lib text file. Sharelib mapping files contains list of key=value. where key is the action key and + * value is the DFS location of sharelib files. * * @param shareLibMap the share lib jar map * @param sharelipFileMapping the sharelip file mapping + * @param symlinkMapping the symlink mapping + * @parm shareLibKey the sharelib key * @throws IOException Signals that an I/O exception has occurred. */ - @SuppressWarnings("unchecked") - private void loadShareLibMetaFile(Map<String, List<Path>> shareLibMap, String sharelibFileMapping) + private void loadShareLibMetaFile(Map<String, List<Path>> shareLibMap, + Map<String, Map<Path, Path>> symlinkMapping, String sharelibFileMapping, String shareLibKey) throws IOException { Path shareFileMappingPath = new Path(sharelibFileMapping); @@ -542,22 +592,34 @@ public class ShareLibService implements Service, Instrumentable { for (Object keyObject : prop.keySet()) { String key = (String) keyObject; - if (key.toLowerCase().startsWith(SHARE_LIB_CONF_PREFIX)) { - String mapKey = key.substring(SHARE_LIB_CONF_PREFIX.length() + 1); - String pathList[] = ((String) prop.get(key)).split(","); - List<Path> listOfPaths = new ArrayList<Path>(); - for (String dfsPath : pathList) { - getPathRecursively(fs, new Path(dfsPath), listOfPaths); - } - shareLibMap.put(mapKey, listOfPaths); - LOG.info("Share lib for " + mapKey + ":" + listOfPaths); - - } - else { - LOG.info(" Not adding " + key + " to sharelib, not prefix with " + SHARE_LIB_CONF_PREFIX); + String mapKey = key.substring(SHARE_LIB_CONF_PREFIX.length() + 1); + if (key.toLowerCase().startsWith(SHARE_LIB_CONF_PREFIX) + && (shareLibKey == null || shareLibKey.equals(mapKey))) { + loadSharelib(shareLibMap, symlinkMapping, mapKey, ((String) prop.get(key)).split(",")); } + } + } + + private void loadSharelib(Map<String, List<Path>> tmpShareLibMap, Map<String, Map<Path, Path>> tmpSymlinkMapping, + String shareLibKey, String pathList[]) throws IOException { + List<Path> listOfPaths = new ArrayList<Path>(); + Map<Path, Path> symlinkMappingforAction = new HashMap<Path, Path>(); + HadoopShims fileSystem = new HadoopShims(fs); + + for (String dfsPath : pathList) { + Path path = new Path(dfsPath); + getPathRecursively(fs, path, listOfPaths); + if (HadoopShims.isSymlinkSupported() && fileSystem.isSymlink(path)) { + symlinkMappingforAction.put(path, fileSystem.getSymLinkTarget(path)); + } + } + if (HadoopShims.isSymlinkSupported()) { + LOG.info("symlink for " + shareLibKey + ":" + symlinkMappingforAction); + tmpSymlinkMapping.put(shareLibKey, symlinkMappingforAction); } + tmpShareLibMap.put(shareLibKey, listOfPaths); + LOG.info("Share lib for " + shareLibKey + ":" + listOfPaths); } /** @@ -683,6 +745,27 @@ public class ShareLibService implements Service, Instrumentable { return getLauncherlibPath().toUri().toString(); } }); + instr.addVariable("libs", "sharelib.symlink.mapping", new Instrumentation.Variable<String>() { + @Override + public String getValue() { + Map<String, Map<Path, Path>> shareLibSymlinkMapping = getSymlinkMapping(); + if (shareLibSymlinkMapping != null && !shareLibSymlinkMapping.isEmpty() + && shareLibSymlinkMapping.values() != null && !shareLibSymlinkMapping.values().isEmpty()) { + StringBuffer bf = new StringBuffer(); + for (Entry<String, Map<Path, Path>> key : shareLibSymlinkMapping.entrySet()) + if (key.getKey() != null && !key.getValue().isEmpty()) { + for (Path path : key.getValue().keySet()) { + bf.append(path).append("(").append(key).append(")").append("=>") + .append(shareLibSymlinkMapping.get(key).get(path)).append(","); + } + + } + return bf.toString(); + } + return "(none)"; + } + }); + } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/b5d409e6/core/src/test/java/org/apache/oozie/service/TestShareLibService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java index 67bb2e5..f261448 100644 --- a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java +++ b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java @@ -40,6 +40,7 @@ import org.apache.oozie.action.hadoop.JavaActionExecutor; import org.apache.oozie.action.hadoop.PigActionExecutor; import org.apache.oozie.action.hadoop.TestJavaActionExecutor; import org.apache.oozie.client.OozieClient; +import org.apache.oozie.hadoop.utils.HadoopShims; import org.apache.oozie.test.XFsTestCase; import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.XConfiguration; @@ -107,15 +108,16 @@ public class TestShareLibService extends XFsTestCase { setSystemProps(); Configuration conf = services.getConf(); conf.set(ShareLibService.FAIL_FAST_ON_STARTUP, "true"); - //Set dummyfile as metafile which doesn't exist. + // Set dummyfile as metafile which doesn't exist. conf.set(ShareLibService.SHARELIB_MAPPING_FILE, String.valueOf(new Date().getTime())); try { services.init(); fail("Should throw exception"); } - catch(Throwable e){ + catch (Throwable e) { assertTrue(e.getMessage().contains("E0104: Could not fully initialize service")); - } finally { + } + finally { services.destroy(); } } @@ -134,7 +136,8 @@ public class TestShareLibService extends XFsTestCase { assertTrue(getFileSystem().exists(launcherPath.get(0))); List<Path> pigLauncherPath = shareLibService.getSystemLibJars("pig"); assertTrue(getFileSystem().exists(pigLauncherPath.get(0))); - } finally { + } + finally { services.destroy(); } } @@ -173,7 +176,8 @@ public class TestShareLibService extends XFsTestCase { else { assertEquals(2, cacheFiles.length); } - } finally { + } + finally { services.destroy(); } } @@ -211,7 +215,8 @@ public class TestShareLibService extends XFsTestCase { else { assertEquals(2, cacheFiles.length); } - } finally { + } + finally { services.destroy(); } } @@ -271,7 +276,8 @@ public class TestShareLibService extends XFsTestCase { else { assertEquals(3, cacheFiles.length); } - } finally { + } + finally { services.destroy(); } } @@ -344,7 +350,8 @@ public class TestShareLibService extends XFsTestCase { assertTrue(fs.exists(noexpirePath)); assertTrue(fs.exists(noexpirePath1)); assertTrue(fs.exists(expirePath)); - } finally { + } + finally { services.destroy(); } } @@ -389,7 +396,8 @@ public class TestShareLibService extends XFsTestCase { assertTrue(fs.exists(noexpirePath1)); assertTrue(fs.exists(expirePath)); assertFalse(fs.exists(expirePath1)); - } finally { + } + finally { services.destroy(); } } @@ -401,7 +409,7 @@ public class TestShareLibService extends XFsTestCase { FileSystem fs = getFileSystem(); Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH)); - //Use basepath if there is no timestamped directory + // Use basepath if there is no timestamped directory fs.mkdirs(basePath); Path pigPath = new Path(basePath.toString() + Path.SEPARATOR + "pig"); fs.mkdirs(pigPath); @@ -409,7 +417,8 @@ public class TestShareLibService extends XFsTestCase { services.init(); ShareLibService shareLibService = Services.get().get(ShareLibService.class); assertNotNull(shareLibService.getShareLibJars("pig")); - } finally { + } + finally { services.destroy(); } } @@ -421,7 +430,7 @@ public class TestShareLibService extends XFsTestCase { FileSystem fs = getFileSystem(); Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH)); - //Use timedstamped directory if available + // Use timedstamped directory if available Date time = new Date(System.currentTimeMillis()); Path libpath = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + ShareLibService.dateFormat.format(time)); fs.mkdirs(libpath); @@ -439,7 +448,8 @@ public class TestShareLibService extends XFsTestCase { assertNotNull(shareLibService.getShareLibJars("pig_9")); assertNotNull(shareLibService.getShareLibJars("pig_10")); assertNull(shareLibService.getShareLibJars("pig_11")); - } finally { + } + finally { services.destroy(); } } @@ -464,7 +474,8 @@ public class TestShareLibService extends XFsTestCase { services.init(); ShareLibService shareLibService = Services.get().get(ShareLibService.class); assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar")); - } finally { + } + finally { services.destroy(); } } @@ -474,7 +485,7 @@ public class TestShareLibService extends XFsTestCase { services = new Services(); FileSystem fs = getFileSystem(); setSystemProps(); - creatTempshareLibKeymap(fs); + createTestShareLibMetaFile(fs); Configuration conf = services.getConf(); conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + "/user/test/config.properties"); conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true"); @@ -484,7 +495,8 @@ public class TestShareLibService extends XFsTestCase { assertTrue(shareLibService.getShareLibJars("something_new").get(0).getName().endsWith("somethingNew.jar")); assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar")); fs.delete(new Path("shareLibPath/"), true); - } finally { + } + finally { services.destroy(); } } @@ -499,7 +511,8 @@ public class TestShareLibService extends XFsTestCase { Date time = new Date(System.currentTimeMillis()); Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH)); - Path libpath = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + ShareLibService.dateFormat.format(time)); + Path libpath = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + + ShareLibService.dateFormat.format(time)); fs.mkdirs(libpath); Path pigPath = new Path(libpath.toString() + Path.SEPARATOR + "pig"); @@ -542,7 +555,8 @@ public class TestShareLibService extends XFsTestCase { else { assertEquals(2, cacheFiles.length); } - } finally { + } + finally { services.destroy(); } } @@ -551,7 +565,7 @@ public class TestShareLibService extends XFsTestCase { public void testShareLibLoadFileMultipleFile() throws Exception { FileSystem fs = getFileSystem(); services = new Services(); - creatTempshareLibKeymap_multipleFile(fs); + createTestShareLibMetaFile_multipleFile(fs); setSystemProps(); Configuration conf = services.getConf(); conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + "/user/test/config.properties"); @@ -562,7 +576,8 @@ public class TestShareLibService extends XFsTestCase { assertNull(shareLibService.getShareLibJars("something_new")); assertEquals(shareLibService.getShareLibJars("pig").size(), 2); fs.delete(new Path("shareLibPath/"), true); - } finally { + } + finally { services.destroy(); } } @@ -574,10 +589,8 @@ public class TestShareLibService extends XFsTestCase { Configuration conf = services.getConf(); conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true"); try { - services.init(); FileSystem fs = getFileSystem(); Date time = new Date(System.currentTimeMillis()); - ShareLibService shareLibService = Services.get().get(ShareLibService.class); Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH)); Path libpath = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + ShareLibService.dateFormat.format(time)); @@ -585,6 +598,8 @@ public class TestShareLibService extends XFsTestCase { Path ooziePath = new Path(libpath.toString() + Path.SEPARATOR + "oozie"); fs.mkdirs(ooziePath); createFile(libpath.toString() + Path.SEPARATOR + "oozie" + Path.SEPARATOR + "oozie_luncher.jar"); + services.init(); + ShareLibService shareLibService = Services.get().get(ShareLibService.class); shareLibService.init(services); List<Path> launcherPath = shareLibService.getSystemLibJars(JavaActionExecutor.OOZIE_COMMON_LIBDIR); assertEquals(launcherPath.size(), 2); @@ -596,13 +611,70 @@ public class TestShareLibService extends XFsTestCase { } } - public void createFile(String filename) throws IOException { + @Test + public void testMetafileSymlink() throws ServiceException, IOException { + // Assume.assumeTrue("Skipping for hadoop - 1.x",HadoopFileSystem.isSymlinkSupported()); + if (!HadoopShims.isSymlinkSupported()) { + return; + } + + services = new Services(); + setSystemProps(); + Configuration conf = services.getConf(); + conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true"); + services.init(); + FileSystem fs = getFileSystem(); + Properties prop = new Properties(); + try { + + String testPath = "shareLibPath/"; + + Path basePath = new Path(testPath + Path.SEPARATOR + "testPath"); + Path basePath1 = new Path(testPath + Path.SEPARATOR + "testPath1"); + Path symlink = new Path("symlink/"); + fs.mkdirs(basePath); + + createFile(basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar"); + createFile(basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_1.jar"); + + createFile(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_2.jar"); + createFile(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_3.jar"); + createFile(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_4.jar"); + + HadoopShims fileSystem = new HadoopShims(fs); + fileSystem.createSymlink(basePath, symlink, true); + + prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig", "/user/test/" + symlink.toString()); + createTestShareLibMetaFile(fs, prop); + assertEquals(fileSystem.isSymlink(symlink), true); + + conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + "/user/test/config.properties"); + conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true"); + try { + ShareLibService shareLibService = Services.get().get(ShareLibService.class); + shareLibService.init(services); + assertEquals(shareLibService.getShareLibJars("pig").size(), 2); + new HadoopShims(fs).createSymlink(basePath1, symlink, true); + assertEquals(shareLibService.getShareLibJars("pig").size(), 3); + } + finally { + fs.delete(new Path("shareLibPath/"), true); + fs.delete(symlink, true); + services.destroy(); + } + } + catch (IOException ex) { + ex.printStackTrace(); + } + } + + private void createFile(String filename) throws IOException { Path path = new Path(filename); FSDataOutputStream out = getFileSystem().create(path); out.close(); } - public void creatTempshareLibKeymap(FileSystem fs) { + private void createTestShareLibMetaFile(FileSystem fs) { Properties prop = new Properties(); try { @@ -620,9 +692,17 @@ public class TestShareLibService extends XFsTestCase { prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig", "/user/test/" + basePath.toString()); prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".something_new", "/user/test/" + somethingNew.toString()); + createTestShareLibMetaFile(fs, prop); - FSDataOutputStream out = fs.create(new Path("/user/test/config.properties")); + } + catch (IOException ex) { + ex.printStackTrace(); + } + } + private void createTestShareLibMetaFile(FileSystem fs, Properties prop) { + try { + FSDataOutputStream out = fs.create(new Path("/user/test/config.properties")); prop.store(out, null); out.close(); @@ -632,7 +712,7 @@ public class TestShareLibService extends XFsTestCase { } } - public void creatTempshareLibKeymap_multipleFile(FileSystem fs) { + public void createTestShareLibMetaFile_multipleFile(FileSystem fs) { Properties prop = new Properties(); try { http://git-wip-us.apache.org/repos/asf/oozie/blob/b5d409e6/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java new file mode 100644 index 0000000..9a19770 --- /dev/null +++ b/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.hadoop.utils; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import java.io.IOException; + +public class HadoopShims { + FileSystem fs; + + public HadoopShims(FileSystem fs) { + this.fs = fs; + } + + public static boolean isSymlinkSupported() { + return false; + } + + public Path getSymLinkTarget(Path p) throws IOException { + return p; + } + + public boolean isSymlink(Path p) throws IOException { + return false; + } + + public void createSymlink(Path target, Path link, boolean createParent) throws IOException { + } + +} http://git-wip-us.apache.org/repos/asf/oozie/blob/b5d409e6/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java new file mode 100644 index 0000000..bea4781 --- /dev/null +++ b/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.hadoop.utils; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import java.io.IOException; + + +public class HadoopShims { + FileSystem fs; + + public HadoopShims(FileSystem fs) { + this.fs = fs; + } + + public static boolean isSymlinkSupported() { + return false; + } + + public Path getSymLinkTarget(Path p) throws IOException { + return p; + } + + public boolean isSymlink(Path p) throws IOException { + return false; + } + + public void createSymlink(Path target, Path link, boolean createParent) throws IOException { + } + +} http://git-wip-us.apache.org/repos/asf/oozie/blob/b5d409e6/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java new file mode 100644 index 0000000..acebd60 --- /dev/null +++ b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.hadoop.utils; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import java.io.IOException; + +public class HadoopShims { + FileSystem fs; + + public HadoopShims(FileSystem fs) { + this.fs = fs; + } + + public static boolean isSymlinkSupported() { + return true; + } + + public Path getSymLinkTarget(Path p) throws IOException { + return fs.getFileLinkStatus(p).getSymlink(); + } + + public boolean isSymlink(Path p) throws IOException { + return fs.getFileLinkStatus(p).isSymlink(); + } + + public void createSymlink(Path target, Path link, boolean createParent) throws IOException { + fs.createSymlink(target, link, createParent); + } + +} http://git-wip-us.apache.org/repos/asf/oozie/blob/b5d409e6/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java new file mode 100644 index 0000000..acebd60 --- /dev/null +++ b/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.hadoop.utils; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import java.io.IOException; + +public class HadoopShims { + FileSystem fs; + + public HadoopShims(FileSystem fs) { + this.fs = fs; + } + + public static boolean isSymlinkSupported() { + return true; + } + + public Path getSymLinkTarget(Path p) throws IOException { + return fs.getFileLinkStatus(p).getSymlink(); + } + + public boolean isSymlink(Path p) throws IOException { + return fs.getFileLinkStatus(p).isSymlink(); + } + + public void createSymlink(Path target, Path link, boolean createParent) throws IOException { + fs.createSymlink(target, link, createParent); + } + +} http://git-wip-us.apache.org/repos/asf/oozie/blob/b5d409e6/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index aa8ef33..e3dad74 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.2.0 release (trunk - unreleased) +OOZIE-2045 Symlink support for sharelib (puru) OOZIE-1385 Make Uber Mode the default (rkanter) OOZIE-1890 Make oozie-site empty and reconcile defaults between oozie-default and the code (seoeun25 via rkanter) OOZIE-2001 Workflow re-runs doesn't update coord action status (jaydeepvishwakarma via shwethags)
