Repository: oozie Updated Branches: refs/heads/master 5045f89cf -> a015a45ee
OOZIE-1761 Improve sharelib purging logic (puru via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a015a45e Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a015a45e Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a015a45e Branch: refs/heads/master Commit: a015a45ee50a23add71ee84e8ef6a6bd00d8aa45 Parents: 5045f89 Author: Rohini Palaniswamy <[email protected]> Authored: Fri Apr 4 12:22:38 2014 -0700 Committer: Rohini Palaniswamy <[email protected]> Committed: Fri Apr 4 12:22:38 2014 -0700 ---------------------------------------------------------------------- .../apache/oozie/service/ShareLibService.java | 45 +++++--- .../oozie/service/TestShareLibService.java | 113 ++++++++++++------- release-log.txt | 1 + 3 files changed, 100 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/a015a45e/core/src/main/java/org/apache/oozie/service/ShareLibService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/ShareLibService.java b/core/src/main/java/org/apache/oozie/service/ShareLibService.java index 89939f1..73ca492 100644 --- a/core/src/main/java/org/apache/oozie/service/ShareLibService.java +++ b/core/src/main/java/org/apache/oozie/service/ShareLibService.java @@ -92,6 +92,8 @@ public class ShareLibService implements Service, Instrumentable { FileSystem fs; + final long retentionTime = 1000 * 60 * 60 * 24 * Services.get().getConf().getInt(LAUNCHERJAR_LIB_RETENTION, 7); + @Override public void init(Services services) throws ServiceException { this.services = services; @@ -106,8 +108,9 @@ public class ShareLibService implements Service, Instrumentable { updateShareLib(); //Only one server should purge sharelib if (Services.get().get(JobsConcurrencyService.class).isFirstServer()) { - purgeLibs(fs, LAUNCHER_PREFIX); - purgeLibs(fs, SHARED_LIB_PREFIX); + final Date current = Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime(); + purgeLibs(fs, LAUNCHER_PREFIX, current); + purgeLibs(fs, SHARED_LIB_PREFIX, current); } } catch (Exception e) { @@ -359,14 +362,26 @@ public class ShareLibService implements Service, Instrumentable { * @throws IOException Signals that an I/O exception has occurred. * @throws ParseException the parse exception */ - private void purgeLibs(FileSystem fs, final String prefix) throws IOException, ParseException { - Configuration conf = services.getConf(); + private void purgeLibs(FileSystem fs, final String prefix, final Date current) throws IOException, ParseException { Path executorLibBasePath = services.get(WorkflowAppService.class).getSystemLibPath(); - PathFilter directoryFilter = new PathFilter() { @Override public boolean accept(Path path) { - return path.getName().startsWith(prefix); + if (path.getName().startsWith(prefix)) { + String name = path.getName().toString(); + String time = name.substring(prefix.length()); + Date d = null; + try { + d = dateFormat.parse(time); + } + catch (ParseException e) { + return false; + } + return (current.getTime() - d.getTime()) > retentionTime; + } + else { + return false; + } } }; FileStatus[] dirList = fs.listStatus(executorLibBasePath, directoryFilter); @@ -376,20 +391,14 @@ public class ShareLibService implements Service, Instrumentable { public int compare(FileStatus o1, FileStatus o2) { return o2.getPath().getName().compareTo(o1.getPath().getName()); } - }); - Date current = Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime(); - // Always keep top two, so start counter from 3 - long retentionTime = 1000 * 60 * 60 * 24 * conf.getInt(LAUNCHERJAR_LIB_RETENTION, 7); - for (int i = 2; i < dirList.length; i++) { + + //Logic is to keep all share-lib between current timestamp and 7days old + 1 latest sharelib older than 7 days. + // refer OOZIE-1761 + for (int i = 1; i < dirList.length; i++) { Path dirPath = dirList[i].getPath(); - String name = dirPath.getName().toString(); - String time = name.substring(prefix.length()); - Date d = dateFormat.parse(time); - if ((current.getTime() - d.getTime()) > retentionTime) { - fs.delete(dirPath, true); - LOG.info("Deleted old launcher jar lib directory {0}", dirPath.getName()); - } + fs.delete(dirPath, true); + LOG.info("Deleted old launcher jar lib directory {0}", dirPath.getName()); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/a015a45e/core/src/test/java/org/apache/oozie/service/TestShareLibService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java index e7d7a26..e1c5c5f 100644 --- a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java +++ b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Properties; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; @@ -50,6 +51,7 @@ public class TestShareLibService extends XFsTestCase { Services services; private static String testCaseDirPath; String shareLibPath = "shareLibPath"; + SimpleDateFormat dt = new SimpleDateFormat("yyyyMMddHHmmss"); @Override protected void setUp() throws Exception { @@ -253,32 +255,29 @@ public class TestShareLibService extends XFsTestCase { Configuration conf = services.getConf(); conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true"); FileSystem fs = getFileSystem(); + long expiryTime = System.currentTimeMillis() + - TimeUnit.MILLISECONDS.convert( + services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7), TimeUnit.DAYS); + // for directory created 8 days back to be deleted - int expire1 = services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7) + 1; + String expireTs = dt.format(new Date(expiryTime - TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS))); // for directory created 6 days back NOT to be deleted - int noexpire = services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7) - 1; + String noexpireTs = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS))); // for directory created 5 days back NOT to be deleted - int noexpire1 = services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7) - 2; - - Date expireDate = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * expire1)); - Date noexpireDate = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * noexpire)); - Date noexpireDate1 = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * noexpire1)); - String expireTs = new SimpleDateFormat("yyyyMMddHHmmss").format(expireDate); - String noexpireTs = new SimpleDateFormat("yyyyMMddHHmmss").format(noexpireDate); - String noexpireTs1 = new SimpleDateFormat("yyyyMMddHHmmss").format(noexpireDate1); + String noexpireTs1 = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS))); + Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH)); Path expirePath = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + expireTs); Path noexpirePath = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + noexpireTs); Path noexpirePath1 = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + noexpireTs1); - fs.mkdirs(expirePath); - fs.mkdirs(noexpirePath); - fs.mkdirs(noexpirePath1); + + createDirs(fs, expirePath, noexpirePath, noexpirePath1); try { services.init(); - assertEquals(3, fs.listStatus(basePath).length); + assertEquals(4, fs.listStatus(basePath).length); assertTrue(fs.exists(noexpirePath)); assertTrue(fs.exists(noexpirePath1)); - assertTrue(!fs.exists(expirePath)); + assertTrue(fs.exists(expirePath)); } finally { services.destroy(); @@ -292,33 +291,66 @@ public class TestShareLibService extends XFsTestCase { Configuration conf = services.getConf(); conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true"); FileSystem fs = getFileSystem(); + + long expiryTime = System.currentTimeMillis() + - TimeUnit.MILLISECONDS.convert( + services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7), TimeUnit.DAYS); + // for directory created 8 days back to be deleted - int expire1 = services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7) + 1; + String expireTs = dt.format(new Date(expiryTime - TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS))); // for directory created 6 days back NOT to be deleted - int noexpire = services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7) - 1; + String noexpireTs = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS))); // for directory created 5 days back NOT to be deleted - int noexpire1 = services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7) - 2; - - Date expireDate = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * expire1)); - Date noexpireDate = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * noexpire)); - Date noexpireDate1 = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * noexpire1)); - String expireTs = new SimpleDateFormat("yyyyMMddHHmmss").format(expireDate); - String noexpireTs = new SimpleDateFormat("yyyyMMddHHmmss").format(noexpireDate); - String noexpireTs1 = new SimpleDateFormat("yyyyMMddHHmmss").format(noexpireDate1); + String noexpireTs1 = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS))); + Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH)); Path expirePath = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + expireTs); Path noexpirePath = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + noexpireTs); Path noexpirePath1 = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + noexpireTs1); + createDirs(fs, expirePath, noexpirePath, noexpirePath1); - fs.mkdirs(expirePath); - fs.mkdirs(noexpirePath); - fs.mkdirs(noexpirePath1); services.init(); - assertEquals(3, fs.listStatus(basePath).length); + assertEquals(4, fs.listStatus(basePath).length); assertTrue(fs.exists(noexpirePath)); assertTrue(fs.exists(noexpirePath1)); - assertTrue(!fs.exists(expirePath)); + assertTrue(fs.exists(expirePath)); + services.destroy(); + } + + // Logic is to keep all share-lib between current timestamp and 7days old + 1 latest sharelib older than 7 days. + // refer OOZIE-1761 + @Test + public void testPurgeJar() throws Exception { + services = new Services(); + setSystemProps(); + Configuration conf = services.getConf(); + conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true"); + FileSystem fs = getFileSystem(); + // for directory created 8 days back to be deleted + long expiryTime = System.currentTimeMillis() + - TimeUnit.MILLISECONDS.convert( + services.getConf().getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7), TimeUnit.DAYS); + + String expireTs = dt.format(new Date(expiryTime - TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS))); + String expireTs1 = dt.format(new Date(expiryTime - TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS))); + String noexpireTs = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS))); + String noexpireTs1 = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS))); + Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH)); + + Path expirePath = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + expireTs); + Path expirePath1 = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + expireTs1); + Path noexpirePath = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + noexpireTs); + Path noexpirePath1 = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + noexpireTs1); + + createDirs(fs, expirePath, expirePath1, noexpirePath, noexpirePath1); + services.init(); + assertEquals(4, fs.listStatus(basePath).length); + assertTrue(fs.exists(noexpirePath)); + assertTrue(fs.exists(noexpirePath1)); + assertTrue(fs.exists(expirePath)); + assertTrue(!fs.exists(expirePath1)); + services.destroy(); } @@ -355,21 +387,14 @@ public class TestShareLibService extends XFsTestCase { Configuration conf = services.getConf(); conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true"); FileSystem fs = getFileSystem(); - - Date day1 = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * 1)); - Date day2 = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * 2)); - Date day3 = new Date(System.currentTimeMillis() - (1000 * 60 * 60 * 24 * 3)); - String dir1 = new SimpleDateFormat("yyyyMMddHHmmss").format(day1); - String dir2 = new SimpleDateFormat("yyyyMMddHHmmss").format(day2); - String dir3 = new SimpleDateFormat("yyyyMMddHHmmss").format(day3); + String dir1 = dt.format(new Date(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS))); + String dir2 = dt.format(new Date(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS))); + String dir3 = dt.format(new Date(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(3, TimeUnit.DAYS))); Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH)); Path path1 = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + dir1); Path path2 = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + dir2); - Path path3 = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + dir3); - fs.mkdirs(path1); - fs.mkdirs(path2); - fs.mkdirs(path3); + createDirs(fs, path1, path2, path3); createFile(path1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar"); services.init(); ShareLibService shareLibService = Services.get().get(ShareLibService.class); @@ -534,4 +559,10 @@ public class TestShareLibService extends XFsTestCase { } } + private void createDirs(FileSystem fs, Path... paths) throws IOException { + for (Path path : paths) { + fs.mkdirs(path); + } + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/a015a45e/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 9673bac..c241774 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.1.0 release (trunk - unreleased) +OOZIE-1761 Improve sharelib purging logic (puru via rohini) OOZIE-1725 add coord EL functions to be used in SLA tag (ryota) OOZIE-1765 JMS Notifications for Workflows not always on the correct topic (rkanter) OOZIE-1732 Sharelib instrumentation fails if sharelib.system.libpath is not created (ryota)
