Repository: oozie Updated Branches: refs/heads/master 65e2109df -> b9c3e0b0c
OOZIE-1783 Sharelib purging only occurs at Oozie startup (rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/b9c3e0b0 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/b9c3e0b0 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/b9c3e0b0 Branch: refs/heads/master Commit: b9c3e0b0c9a62094d2a53034de53d6b4cc62b7c2 Parents: 65e2109 Author: Robert Kanter <[email protected]> Authored: Thu May 22 17:19:47 2014 -0700 Committer: Robert Kanter <[email protected]> Committed: Thu May 22 17:19:47 2014 -0700 ---------------------------------------------------------------------- .../apache/oozie/service/ShareLibService.java | 36 +- core/src/main/resources/oozie-default.xml | 8 + .../oozie/service/TestShareLibService.java | 401 ++++++++++--------- release-log.txt | 1 + .../oozie/tools/TestOozieSharelibCLI.java | 7 +- 5 files changed, 254 insertions(+), 199 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/b9c3e0b0/core/src/main/java/org/apache/oozie/service/ShareLibService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/ShareLibService.java b/core/src/main/java/org/apache/oozie/service/ShareLibService.java index 353b382..320af8b 100644 --- a/core/src/main/java/org/apache/oozie/service/ShareLibService.java +++ b/core/src/main/java/org/apache/oozie/service/ShareLibService.java @@ -63,6 +63,8 @@ public class ShareLibService implements Service, Instrumentable { public static final String SHIP_LAUNCHER_JAR = "oozie.action.ship.launcher.jar"; + public static final String PURGE_INTERVAL = CONF_PREFIX + "ShareLibService.purge.interval"; + private static final String PERMISSION_STRING = "-rwxr-xr-x"; public static final String LAUNCHER_PREFIX = "launcher_"; @@ -114,18 +116,25 @@ public class ShareLibService implements Service, Instrumentable { + "'oozie admin' CLI command to update the sharelib", ioe); LOG.error(se); } - try { - //Only one server should purge sharelib - if (Services.get().get(JobsConcurrencyService.class).isFirstServer()) { - final Date current = Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime(); - purgeLibs(fs, LAUNCHER_PREFIX, current); - purgeLibs(fs, SHARED_LIB_PREFIX, current); + Runnable purgeLibsRunnable = new Runnable() { + @Override + public void run() { + System.out.flush(); + try { + //Only one server should purge sharelib + if (Services.get().get(JobsConcurrencyService.class).isFirstServer()) { + final Date current = Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime(); + purgeLibs(fs, LAUNCHER_PREFIX, current); + purgeLibs(fs, SHARED_LIB_PREFIX, current); + } + } + catch (IOException e) { + LOG.error("There was an issue purging the sharelib", e); + } } - } - catch (Exception e) { - LOG.error("There was an issue purging the sharelib", e); - } - + }; + services.get(SchedulerService.class).schedule(purgeLibsRunnable, 10, + services.getConf().getInt(PURGE_INTERVAL, 1) * 60 * 60 * 24, SchedulerService.Unit.SEC); } /** @@ -369,15 +378,14 @@ public class ShareLibService implements Service, Instrumentable { * @param fs the fs * @param prefix the prefix * @throws IOException Signals that an I/O exception has occurred. - * @throws ParseException the parse exception */ - private void purgeLibs(FileSystem fs, final String prefix, final Date current) throws IOException, ParseException { + private void purgeLibs(FileSystem fs, final String prefix, final Date current) throws IOException { Path executorLibBasePath = services.get(WorkflowAppService.class).getSystemLibPath(); PathFilter directoryFilter = new PathFilter() { @Override public boolean accept(Path path) { if (path.getName().startsWith(prefix)) { - String name = path.getName().toString(); + String name = path.getName(); String time = name.substring(prefix.length()); Date d = null; try { http://git-wip-us.apache.org/repos/asf/oozie/blob/b9c3e0b0/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 9788daf..63a91fa 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -2048,6 +2048,14 @@ </property> <property> + <name>oozie.service.ShareLibService.purge.interval</name> + <value>1</value> + <description> + How often, in days, Oozie should check for old ShareLibs and LauncherLibs to purge from HDFS. + </description> + </property> + + <property> <name>oozie.action.jobinfo.enable</name> <value>false</value> <description> http://git-wip-us.apache.org/repos/asf/oozie/blob/b9c3e0b0/core/src/test/java/org/apache/oozie/service/TestShareLibService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java index e1c5c5f..5fcbbda 100644 --- a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java +++ b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java @@ -106,16 +106,17 @@ public class TestShareLibService extends XFsTestCase { setSystemProps(); Configuration conf = services.getConf(); conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true"); - services.init(); - ShareLibService shareLibService = Services.get().get(ShareLibService.class); - List<Path> launcherPath = shareLibService.getSystemLibJars(JavaActionExecutor.OOZIE_COMMON_LIBDIR); - assertNotNull(launcherPath); - assertTrue(getFileSystem().exists(launcherPath.get(0))); - List<Path> pigLauncherPath = shareLibService.getSystemLibJars("pig"); - assertTrue(getFileSystem().exists(pigLauncherPath.get(0))); - - services.destroy(); - + try { + services.init(); + ShareLibService shareLibService = Services.get().get(ShareLibService.class); + List<Path> launcherPath = shareLibService.getSystemLibJars(JavaActionExecutor.OOZIE_COMMON_LIBDIR); + assertNotNull(launcherPath); + assertTrue(getFileSystem().exists(launcherPath.get(0))); + List<Path> pigLauncherPath = shareLibService.getSystemLibJars("pig"); + assertTrue(getFileSystem().exists(pigLauncherPath.get(0))); + } finally { + services.destroy(); + } } @Test @@ -125,33 +126,36 @@ public class TestShareLibService extends XFsTestCase { Configuration conf = services.getConf(); conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true"); - services.init(); - String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" - + getNameNodeUri() + "</name-node>" + "</java>"; - Element eActionXml = XmlUtils.parseXml(actionXml); - XConfiguration protoConf = new XConfiguration(); - protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); - WorkflowJobBean wfj = new WorkflowJobBean(); - wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString()); - wfj.setConf(XmlUtils.prettyPrint(new XConfiguration()).toString()); - Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean()); - PigActionExecutor ae = new PigActionExecutor(); - Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml); - ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf); - - URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf); - String cacheFilesStr = Arrays.toString(cacheFiles); - assertTrue(cacheFilesStr.contains(MyPig.class.getName() + ".jar")); - assertTrue(cacheFilesStr.contains(MyOozie.class.getName() + ".jar")); - // Hadoop 2 has two extra jars - if (cacheFiles.length == 4) { - assertTrue(cacheFilesStr.contains("MRAppJar.jar")); - assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-")); - } - else { - assertEquals(2, cacheFiles.length); + try { + services.init(); + String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + + getNameNodeUri() + "</name-node>" + "</java>"; + Element eActionXml = XmlUtils.parseXml(actionXml); + XConfiguration protoConf = new XConfiguration(); + protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); + WorkflowJobBean wfj = new WorkflowJobBean(); + wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString()); + wfj.setConf(XmlUtils.prettyPrint(new XConfiguration()).toString()); + Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean()); + PigActionExecutor ae = new PigActionExecutor(); + Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml); + ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf); + + URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf); + String cacheFilesStr = Arrays.toString(cacheFiles); + assertTrue(cacheFilesStr.contains(MyPig.class.getName() + ".jar")); + assertTrue(cacheFilesStr.contains(MyOozie.class.getName() + ".jar")); + // Hadoop 2 has two extra jars + if (cacheFiles.length == 4) { + assertTrue(cacheFilesStr.contains("MRAppJar.jar")); + assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-")); + } + else { + assertEquals(2, cacheFiles.length); + } + } finally { + services.destroy(); } - services.destroy(); } @Test @@ -160,34 +164,36 @@ public class TestShareLibService extends XFsTestCase { setSystemProps(); Configuration conf = services.getConf(); conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true"); - services.init(); - String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" - + getNameNodeUri() + "</name-node>" + "</pig>"; - Element eActionXml = XmlUtils.parseXml(actionXml); - XConfiguration protoConf = new XConfiguration(); - protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); - WorkflowJobBean wfj = new WorkflowJobBean(); - wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString()); - wfj.setConf(XmlUtils.prettyPrint(new XConfiguration()).toString()); - Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean()); - PigActionExecutor ae = new PigActionExecutor(); - Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml); - ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf); - - URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf); - String cacheFilesStr = Arrays.toString(cacheFiles); - assertTrue(cacheFilesStr.contains("MyPig.jar")); - assertTrue(cacheFilesStr.contains("MyOozie.jar")); - // Hadoop 2 has two extra jars - if (cacheFiles.length == 4) { - assertTrue(cacheFilesStr.contains("MRAppJar.jar")); - assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-")); - } - else { - assertEquals(2, cacheFiles.length); + try { + services.init(); + String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + + getNameNodeUri() + "</name-node>" + "</pig>"; + Element eActionXml = XmlUtils.parseXml(actionXml); + XConfiguration protoConf = new XConfiguration(); + protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); + WorkflowJobBean wfj = new WorkflowJobBean(); + wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString()); + wfj.setConf(XmlUtils.prettyPrint(new XConfiguration()).toString()); + Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean()); + PigActionExecutor ae = new PigActionExecutor(); + Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml); + ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf); + + URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf); + String cacheFilesStr = Arrays.toString(cacheFiles); + assertTrue(cacheFilesStr.contains("MyPig.jar")); + assertTrue(cacheFilesStr.contains("MyOozie.jar")); + // Hadoop 2 has two extra jars + if (cacheFiles.length == 4) { + assertTrue(cacheFilesStr.contains("MRAppJar.jar")); + assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-")); + } + else { + assertEquals(2, cacheFiles.length); + } + } finally { + services.destroy(); } - services.destroy(); - } @Test @@ -213,39 +219,41 @@ public class TestShareLibService extends XFsTestCase { createFile(libpath.toString() + Path.SEPARATOR + "pig_10" + Path.SEPARATOR + "pig-10.jar"); - services.init(); - String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" - + getNameNodeUri() + "</name-node>" - + "<property><name>oozie.action.sharelib.for.pig</name><value>pig_10</value></property>" + "</pig>"; - Element eActionXml = XmlUtils.parseXml(actionXml); - XConfiguration protoConf = new XConfiguration(); - protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); - WorkflowJobBean wfj = new WorkflowJobBean(); - protoConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true); - wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString()); - wfj.setConf(XmlUtils.prettyPrint(protoConf).toString()); - - Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean()); - PigActionExecutor ae = new PigActionExecutor(); - Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml); - jobConf.set("oozie.action.sharelib.for.pig", "pig_10"); - ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf); - - URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf); - String cacheFilesStr = Arrays.toString(cacheFiles); - assertTrue(cacheFilesStr.contains("MyPig.jar")); - assertTrue(cacheFilesStr.contains("MyOozie.jar")); - assertTrue(cacheFilesStr.contains("pig-10.jar")); - // Hadoop 2 has two extra jars - if (cacheFiles.length == 5) { - assertTrue(cacheFilesStr.contains("MRAppJar.jar")); - assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-")); - } - else { - assertEquals(3, cacheFiles.length); + try { + services.init(); + String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + + getNameNodeUri() + "</name-node>" + + "<property><name>oozie.action.sharelib.for.pig</name><value>pig_10</value></property>" + "</pig>"; + Element eActionXml = XmlUtils.parseXml(actionXml); + XConfiguration protoConf = new XConfiguration(); + protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); + WorkflowJobBean wfj = new WorkflowJobBean(); + protoConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true); + wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString()); + wfj.setConf(XmlUtils.prettyPrint(protoConf).toString()); + + Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean()); + PigActionExecutor ae = new PigActionExecutor(); + Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml); + jobConf.set("oozie.action.sharelib.for.pig", "pig_10"); + ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf); + + URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf); + String cacheFilesStr = Arrays.toString(cacheFiles); + assertTrue(cacheFilesStr.contains("MyPig.jar")); + assertTrue(cacheFilesStr.contains("MyOozie.jar")); + assertTrue(cacheFilesStr.contains("pig-10.jar")); + // Hadoop 2 has two extra jars + if (cacheFiles.length == 5) { + assertTrue(cacheFilesStr.contains("MRAppJar.jar")); + assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-")); + } + else { + assertEquals(3, cacheFiles.length); + } + } finally { + services.destroy(); } - services.destroy(); - } @Test @@ -310,12 +318,15 @@ public class TestShareLibService extends XFsTestCase { Path noexpirePath1 = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + noexpireTs1); createDirs(fs, expirePath, noexpirePath, noexpirePath1); - services.init(); - assertEquals(4, fs.listStatus(basePath).length); - assertTrue(fs.exists(noexpirePath)); - assertTrue(fs.exists(noexpirePath1)); - assertTrue(fs.exists(expirePath)); - services.destroy(); + try { + services.init(); + assertEquals(4, fs.listStatus(basePath).length); + assertTrue(fs.exists(noexpirePath)); + assertTrue(fs.exists(noexpirePath1)); + assertTrue(fs.exists(expirePath)); + } finally { + services.destroy(); + } } // Logic is to keep all share-lib between current timestamp and 7days old + 1 latest sharelib older than 7 days. @@ -326,7 +337,7 @@ public class TestShareLibService extends XFsTestCase { setSystemProps(); Configuration conf = services.getConf(); conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true"); - FileSystem fs = getFileSystem(); + final FileSystem fs = getFileSystem(); // for directory created 8 days back to be deleted long expiryTime = System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert( @@ -336,7 +347,7 @@ public class TestShareLibService extends XFsTestCase { String expireTs1 = dt.format(new Date(expiryTime - TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS))); String noexpireTs = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS))); String noexpireTs1 = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS))); - Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH)); + final Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH)); Path expirePath = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + expireTs); Path expirePath1 = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + expireTs1); @@ -344,14 +355,23 @@ public class TestShareLibService extends XFsTestCase { Path noexpirePath1 = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + noexpireTs1); createDirs(fs, expirePath, expirePath1, noexpirePath, noexpirePath1); - services.init(); - assertEquals(4, fs.listStatus(basePath).length); - assertTrue(fs.exists(noexpirePath)); - assertTrue(fs.exists(noexpirePath1)); - assertTrue(fs.exists(expirePath)); - assertTrue(!fs.exists(expirePath1)); - - services.destroy(); + try { + services.init(); + // Wait for the scheduled purge runnable to complete + waitFor(20 * 1000, new Predicate() { + @Override + public boolean evaluate() throws Exception { + return (fs.listStatus(basePath).length == 4); + } + }); + assertEquals(4, fs.listStatus(basePath).length); + assertTrue(fs.exists(noexpirePath)); + assertTrue(fs.exists(noexpirePath1)); + assertTrue(fs.exists(expirePath)); + assertFalse(fs.exists(expirePath1)); + } finally { + services.destroy(); + } } @Test @@ -371,13 +391,16 @@ public class TestShareLibService extends XFsTestCase { fs.mkdirs(pigPath); fs.mkdirs(pigPath1); fs.mkdirs(pigPath2); - services.init(); - ShareLibService shareLibService = Services.get().get(ShareLibService.class); - assertNotNull(shareLibService.getShareLibJars("pig")); - assertNotNull(shareLibService.getShareLibJars("pig_9")); - assertNotNull(shareLibService.getShareLibJars("pig_10")); - assertNull(shareLibService.getShareLibJars("pig_11")); - services.destroy(); + try { + services.init(); + ShareLibService shareLibService = Services.get().get(ShareLibService.class); + assertNotNull(shareLibService.getShareLibJars("pig")); + assertNotNull(shareLibService.getShareLibJars("pig_9")); + assertNotNull(shareLibService.getShareLibJars("pig_10")); + assertNull(shareLibService.getShareLibJars("pig_11")); + } finally { + services.destroy(); + } } @Test @@ -396,15 +419,17 @@ public class TestShareLibService extends XFsTestCase { Path path3 = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + dir3); createDirs(fs, path1, path2, path3); createFile(path1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar"); - services.init(); - ShareLibService shareLibService = Services.get().get(ShareLibService.class); - assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar")); - services.destroy(); + try { + services.init(); + ShareLibService shareLibService = Services.get().get(ShareLibService.class); + assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar")); + } finally { + services.destroy(); + } } @Test public void testShareLibLoadFile() throws Exception { - services = new Services(); FileSystem fs = getFileSystem(); setSystemProps(); @@ -412,68 +437,73 @@ public class TestShareLibService extends XFsTestCase { Configuration conf = services.getConf(); conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + "/user/test/config.properties"); conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true"); - services.init(); - ShareLibService shareLibService = Services.get().get(ShareLibService.class); - assertTrue(shareLibService.getShareLibJars("something_new").get(0).getName().endsWith("somethingNew.jar")); - assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar")); - fs.delete(new Path("shareLibPath/"), true); - services.destroy(); + try { + services.init(); + ShareLibService shareLibService = Services.get().get(ShareLibService.class); + assertTrue(shareLibService.getShareLibJars("something_new").get(0).getName().endsWith("somethingNew.jar")); + assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar")); + fs.delete(new Path("shareLibPath/"), true); + } finally { + services.destroy(); + } } @Test public void testLoadfromDFS() throws Exception { services = new Services(); setSystemProps(); - services.init(); - FileSystem fs = getFileSystem(); - Date time = new Date(System.currentTimeMillis()); - - Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH)); - Path libpath = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + ShareLibService.dateFormat.format(time)); - fs.mkdirs(libpath); - - Path pigPath = new Path(libpath.toString() + Path.SEPARATOR + "pig"); - Path ooziePath = new Path(libpath.toString() + Path.SEPARATOR + "oozie"); - Path pigPath1 = new Path(libpath.toString() + Path.SEPARATOR + "pig_9"); - Path pigPath2 = new Path(libpath.toString() + Path.SEPARATOR + "pig_10"); - fs.mkdirs(pigPath); - fs.mkdirs(ooziePath); - fs.mkdirs(pigPath1); - fs.mkdirs(pigPath2); - - createFile(libpath.toString() + Path.SEPARATOR + "pig_10" + Path.SEPARATOR + "pig-10.jar"); - createFile(libpath.toString() + Path.SEPARATOR + "oozie" + Path.SEPARATOR + "oozie_luncher.jar"); - - String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" - + getNameNodeUri() + "</name-node>" - + "<property><name>oozie.action.sharelib.for.pig</name><value>pig_10</value></property>" + "</pig>"; - Element eActionXml = XmlUtils.parseXml(actionXml); - XConfiguration protoConf = new XConfiguration(); - protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); - WorkflowJobBean wfj = new WorkflowJobBean(); - protoConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true); - wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString()); - wfj.setConf(XmlUtils.prettyPrint(protoConf).toString()); - - Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean()); - PigActionExecutor ae = new PigActionExecutor(); - Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml); - jobConf.set("oozie.action.sharelib.for.pig", "pig_10"); - ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf); - URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf); - String cacheFilesStr = Arrays.toString(cacheFiles); - assertTrue(cacheFilesStr.contains("pig-10.jar")); - assertTrue(cacheFilesStr.contains("oozie_luncher.jar")); - // Hadoop 2 has two extra jars - if (cacheFiles.length == 4) { - assertTrue(cacheFilesStr.contains("MRAppJar.jar")); - assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-")); - } - else { - assertEquals(2, cacheFiles.length); + try { + services.init(); + FileSystem fs = getFileSystem(); + Date time = new Date(System.currentTimeMillis()); + + Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH)); + Path libpath = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + ShareLibService.dateFormat.format(time)); + fs.mkdirs(libpath); + + Path pigPath = new Path(libpath.toString() + Path.SEPARATOR + "pig"); + Path ooziePath = new Path(libpath.toString() + Path.SEPARATOR + "oozie"); + Path pigPath1 = new Path(libpath.toString() + Path.SEPARATOR + "pig_9"); + Path pigPath2 = new Path(libpath.toString() + Path.SEPARATOR + "pig_10"); + fs.mkdirs(pigPath); + fs.mkdirs(ooziePath); + fs.mkdirs(pigPath1); + fs.mkdirs(pigPath2); + + createFile(libpath.toString() + Path.SEPARATOR + "pig_10" + Path.SEPARATOR + "pig-10.jar"); + createFile(libpath.toString() + Path.SEPARATOR + "oozie" + Path.SEPARATOR + "oozie_luncher.jar"); + + String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + + getNameNodeUri() + "</name-node>" + + "<property><name>oozie.action.sharelib.for.pig</name><value>pig_10</value></property>" + "</pig>"; + Element eActionXml = XmlUtils.parseXml(actionXml); + XConfiguration protoConf = new XConfiguration(); + protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); + WorkflowJobBean wfj = new WorkflowJobBean(); + protoConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true); + wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString()); + wfj.setConf(XmlUtils.prettyPrint(protoConf).toString()); + + Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean()); + PigActionExecutor ae = new PigActionExecutor(); + Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml); + jobConf.set("oozie.action.sharelib.for.pig", "pig_10"); + ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf); + URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf); + String cacheFilesStr = Arrays.toString(cacheFiles); + assertTrue(cacheFilesStr.contains("pig-10.jar")); + assertTrue(cacheFilesStr.contains("oozie_luncher.jar")); + // Hadoop 2 has two extra jars + if (cacheFiles.length == 4) { + assertTrue(cacheFilesStr.contains("MRAppJar.jar")); + assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-")); + } + else { + assertEquals(2, cacheFiles.length); + } + } finally { + services.destroy(); } - services.destroy(); - } @Test @@ -485,12 +515,15 @@ public class TestShareLibService extends XFsTestCase { Configuration conf = services.getConf(); conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + "/user/test/config.properties"); conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true"); - services.init(); - ShareLibService shareLibService = Services.get().get(ShareLibService.class); - assertNull(shareLibService.getShareLibJars("something_new")); - assertEquals(shareLibService.getShareLibJars("pig").size(), 2); - fs.delete(new Path("shareLibPath/"), true); - services.destroy(); + try { + services.init(); + ShareLibService shareLibService = Services.get().get(ShareLibService.class); + assertNull(shareLibService.getShareLibJars("something_new")); + assertEquals(shareLibService.getShareLibJars("pig").size(), 2); + fs.delete(new Path("shareLibPath/"), true); + } finally { + services.destroy(); + } } public void createFile(String filename) throws IOException { http://git-wip-us.apache.org/repos/asf/oozie/blob/b9c3e0b0/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 8940746..81e26ef 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.1.0 release (trunk - unreleased) +OOZIE-1783 Sharelib purging only occurs at Oozie startup (rkanter) OOZIE-1689 HA support for OOZIE-7(Ability to view the log information corresponding to particular coordinator action) (puru via mona) OOZIE-1849 If the underlying job finishes while a Workflow is suspended, Oozie can take a while to realize it (rkanter) OOZIE-1835 NullPointerException from SLAEmailEventListener (rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/b9c3e0b0/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java ---------------------------------------------------------------------- diff --git a/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java index b2588b8..4c095ef 100644 --- a/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java +++ b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java @@ -65,6 +65,9 @@ public class TestOozieSharelibCLI extends XTestCase { @AfterClass protected void tearDown() throws Exception { System.setSecurityManager(SECURITY_MANAGER); + if (services != null) { + services.destroy(); + } super.tearDown(); } @@ -160,7 +163,9 @@ public class TestOozieSharelibCLI extends XTestCase { services = new Services(); services.getConf() .set(Services.CONF_SERVICE_CLASSES,"org.apache.oozie.service.LiteWorkflowAppService," - + "org.apache.oozie.service.HadoopAccessorService,org.apache.oozie.service.ShareLibService"); + + "org.apache.oozie.service.SchedulerService," + + "org.apache.oozie.service.HadoopAccessorService," + + "org.apache.oozie.service.ShareLibService"); services.init(); } return services;
