Repository: oozie Updated Branches: refs/heads/master 05fce1f78 -> e94a345ac
OOZIE-2312 oozie doesn't purge audit and error log Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/e94a345a Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/e94a345a Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/e94a345a Branch: refs/heads/master Commit: e94a345acf220c04ee870515ad38adb9865c2720 Parents: 05fce1f Author: Purshotam Shah <[email protected]> Authored: Fri Jan 29 09:27:18 2016 -0800 Committer: Purshotam Shah <[email protected]> Committed: Fri Jan 29 09:27:18 2016 -0800 ---------------------------------------------------------------------- core/src/main/conf/oozie-log4j.properties | 7 +- .../java/org/apache/oozie/service/XLogUtil.java | 5 + .../apache/oozie/util/OozieRollingPolicy.java | 109 ++++++++++-------- .../oozie/util/TestOozieRollingPolicy.java | 112 ++++++++++--------- release-log.txt | 1 + 5 files changed, 130 insertions(+), 104 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/e94a345a/core/src/main/conf/oozie-log4j.properties ---------------------------------------------------------------------- diff --git a/core/src/main/conf/oozie-log4j.properties b/core/src/main/conf/oozie-log4j.properties index 2b20ff2..c563687 100644 --- a/core/src/main/conf/oozie-log4j.properties +++ b/core/src/main/conf/oozie-log4j.properties @@ -86,12 +86,15 @@ log4j.appender.oozieinstrumentation.Append=true log4j.appender.oozieinstrumentation.layout=org.apache.log4j.PatternLayout log4j.appender.oozieinstrumentation.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n -log4j.appender.oozieaudit=org.apache.log4j.DailyRollingFileAppender -log4j.appender.oozieaudit.DatePattern='.'yyyy-MM-dd +log4j.appender.oozieaudit=org.apache.log4j.rolling.RollingFileAppender +log4j.appender.oozieaudit.RollingPolicy=org.apache.oozie.util.OozieRollingPolicy log4j.appender.oozieaudit.File=${oozie.log.dir}/oozie-audit.log log4j.appender.oozieaudit.Append=true log4j.appender.oozieaudit.layout=org.apache.log4j.PatternLayout log4j.appender.oozieaudit.layout.ConversionPattern=%d{ISO8601} %5p %c{1}:%L - %m%n +log4j.appender.oozieaudit.RollingPolicy.FileNamePattern=${log4j.appender.oozieaudit.File}.%d{yyyy-MM-dd} +log4j.appender.oozieaudit.RollingPolicy.MaxHistory=30 + log4j.appender.openjpa=org.apache.log4j.DailyRollingFileAppender log4j.appender.openjpa.DatePattern='.'yyyy-MM-dd http://git-wip-us.apache.org/repos/asf/oozie/blob/e94a345a/core/src/main/java/org/apache/oozie/service/XLogUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/XLogUtil.java b/core/src/main/java/org/apache/oozie/service/XLogUtil.java index 59a4d22..aae00c2 100644 --- a/core/src/main/java/org/apache/oozie/service/XLogUtil.java +++ b/core/src/main/java/org/apache/oozie/service/XLogUtil.java @@ -96,6 +96,11 @@ public class XLogUtil { if (pattern.matches(Pattern.quote(logFile) + ".*-%d\\{yyyy-MM-dd-HH\\}(\\.gz)?")) { logRotation = 60 * 60; } + else if (pattern.matches(Pattern.quote(logFile) + ".*-%d\\{yyyy-MM-dd\\}(\\.gz)?") + || pattern.matches(Pattern.quote(logFile) + ".*\\.%d\\{yyyy-MM-dd\\}(\\.gz)?")) { + logRotation = 60 * 60 * 24; + } + else { log.warn( "Oozie WS " http://git-wip-us.apache.org/repos/asf/oozie/blob/e94a345a/core/src/main/java/org/apache/oozie/util/OozieRollingPolicy.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/OozieRollingPolicy.java b/core/src/main/java/org/apache/oozie/util/OozieRollingPolicy.java index 19ce7f9..625053c 100644 --- a/core/src/main/java/org/apache/oozie/util/OozieRollingPolicy.java +++ b/core/src/main/java/org/apache/oozie/util/OozieRollingPolicy.java @@ -19,17 +19,21 @@ package org.apache.oozie.util; import java.io.File; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Calendar; import java.util.Collections; +import java.util.List; import java.util.concurrent.Semaphore; -import java.util.regex.Matcher; + import org.apache.log4j.Appender; +import org.apache.log4j.pattern.ExtrasPatternParser; import org.apache.log4j.rolling.RollingPolicyBase; import org.apache.log4j.rolling.RolloverDescription; import org.apache.log4j.rolling.TimeBasedRollingPolicy; import org.apache.log4j.rolling.TriggeringPolicy; import org.apache.log4j.spi.LoggingEvent; +import org.apache.log4j.pattern.LiteralPatternConverter; import org.apache.oozie.service.Services; import org.apache.oozie.service.XLogService; @@ -49,7 +53,10 @@ public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringP private Thread deleteThread; private int maxHistory = 720; // (720 hours / 24 hours per day = 30 days) as default - + + String oozieLogDir; + String logFileName; + public int getMaxHistory() { return maxHistory; } @@ -63,7 +70,28 @@ public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringP deleteThread = new Thread(); tbrp = new TimeBasedRollingPolicy(); } - + + + @SuppressWarnings("rawtypes") + public void setFileNamePattern(String fnp) { + super.setFileNamePattern(fnp); + List converters = new ArrayList(); + StringBuffer file = new StringBuffer(); + ExtrasPatternParser + .parse(fnp, converters, new ArrayList(), null, ExtrasPatternParser.getFileNamePatternRules()); + if (!converters.isEmpty()) { + ((LiteralPatternConverter) converters.get(0)).format(null, file); + File f = new File(file.toString()); + oozieLogDir = f.getParent(); + logFileName = f.getName(); + } + else { + XLogService xls = Services.get().get(XLogService.class); + oozieLogDir = xls.getOozieLogPath(); + logFileName = xls.getOozieLogName(); + } + + } @Override public void activateOptions() { super.activateOptions(); @@ -106,34 +134,30 @@ public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringP private void deleteOldFiles() { ArrayList<FileInfo> fileList = new ArrayList<FileInfo>(); - XLogService xls = getXLogService(); - if (xls != null) { // We need this to get the paths - String oozieLogPath = xls.getOozieLogPath(); - String logFile = xls.getOozieLogName(); - if (oozieLogPath != null && logFile != null) { - String[] children = new File(oozieLogPath).list(); - if (children != null) { - for (String child : children) { - if (child.startsWith(logFile) && !child.equals(logFile)) { - File childFile = new File(new File(oozieLogPath).getAbsolutePath(), child); - if (child.endsWith(".gz")) { - long gzFileCreationTime = getGZFileCreationTime(child); - if (gzFileCreationTime != -1) { - fileList.add(new FileInfo(childFile.getAbsolutePath(), gzFileCreationTime)); - } - } else{ - long modTime = childFile.lastModified(); - fileList.add(new FileInfo(childFile.getAbsolutePath(), modTime)); + if (oozieLogDir != null && logFileName != null) { + String[] children = new File(oozieLogDir).list(); + if (children != null) { + for (String child : children) { + if (child.startsWith(logFileName) && !child.equals(logFileName)) { + File childFile = new File(new File(oozieLogDir).getAbsolutePath(), child); + if (child.endsWith(".gz")) { + long gzFileCreationTime = getGZFileCreationTime(child); + if (gzFileCreationTime != -1) { + fileList.add(new FileInfo(childFile.getAbsolutePath(), gzFileCreationTime)); } } + else { + long modTime = childFile.lastModified(); + fileList.add(new FileInfo(childFile.getAbsolutePath(), modTime)); + } } } } } - + if (fileList.size() > maxHistory) { Collections.sort(fileList); - + for (int i = maxHistory; i < fileList.size(); i++) { new File(fileList.get(i).getFileName()).delete(); } @@ -141,23 +165,22 @@ public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringP } private long getGZFileCreationTime(String fileName) { - // Default return value of -1 to exclude the file - long returnVal = -1; - Matcher m = XLogStreamer.gzTimePattern.matcher(fileName); - if (m.matches() && m.groupCount() == 4) { - int year = Integer.parseInt(m.group(1)); - int month = Integer.parseInt(m.group(2)); - int day = Integer.parseInt(m.group(3)); - int hour = Integer.parseInt(m.group(4)); - int minute = 0; - Calendar calendarEntry = Calendar.getInstance(); - calendarEntry.set(year, month - 1, day, hour, minute); // give month-1(Say, 7 for August) - long logFileStartTime = calendarEntry.getTimeInMillis(); - returnVal = logFileStartTime; + SimpleDateFormat formatter; + String date = fileName.substring(logFileName.length(), fileName.length() - 3); //3 for .gz + if (date.length() == 10) { + formatter = new SimpleDateFormat("yyyy-MM-dd"); + } + else { + formatter = new SimpleDateFormat("yyyy-MM-dd-HH"); + } + try { + return formatter.parse(date).getTime(); + } + catch (ParseException e) { + return -1; } - return returnVal; } - + class FileInfo implements Comparable<FileInfo> { String fileName; long modTime; @@ -190,12 +213,4 @@ public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringP } } - // Needed for TestOozieRollingPolicy tests to be able to override getOozieLogPath() and getOozieLogName() - // by overriding getXLogService() - XLogService getXLogService() { - if (Services.get() != null) { - return Services.get().get(XLogService.class); - } - return null; - } } http://git-wip-us.apache.org/repos/asf/oozie/blob/e94a345a/core/src/test/java/org/apache/oozie/util/TestOozieRollingPolicy.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/util/TestOozieRollingPolicy.java b/core/src/test/java/org/apache/oozie/util/TestOozieRollingPolicy.java index 9ae7bec..051ce56 100644 --- a/core/src/test/java/org/apache/oozie/util/TestOozieRollingPolicy.java +++ b/core/src/test/java/org/apache/oozie/util/TestOozieRollingPolicy.java @@ -19,11 +19,12 @@ package org.apache.oozie.util; import java.io.File; +import java.io.IOException; import java.util.Calendar; import java.util.GregorianCalendar; + import org.apache.commons.logging.LogFactory; import org.apache.log4j.LogManager; -import org.apache.oozie.service.XLogService; import org.apache.oozie.test.XTestCase; public class TestOozieRollingPolicy extends XTestCase { @@ -41,40 +42,56 @@ public class TestOozieRollingPolicy extends XTestCase { LogManager.resetConfiguration(); super.tearDown(); } - + public void testDeletingOldFiles() throws Exception { + _testDeletingOldFiles("oozie.log"); + } + + public void testDeletingErrorOldFiles() throws Exception { + _testDeletingOldFiles("oozie-error.log"); + } + + public void testDeletingAuditOldFiles() throws Exception { + _testDeletingOldFiles("oozie-audit.log", Calendar.DAY_OF_MONTH); + } + + private void _testDeletingOldFiles(String oozieLogName) throws IOException { + _testDeletingOldFiles(oozieLogName, Calendar.HOUR_OF_DAY); + } + + private void _testDeletingOldFiles(String oozieLogName, int calendarUnit) throws IOException{ // OozieRollingPolicy gets the log path and log name from XLogService by calling Services.get.get(XLogService.class) so we // use a mock version where we overwrite the XLogService.getOozieLogName() and XLogService.getOozieLogPath() to simply // return these values instead of involving Services. We then overwrite OozieRollingPolicy.getXLogService() to return the // mock one instead. String oozieLogPath = getTestCaseDir(); - String oozieLogName = "oozie.log"; - final XLogService xls = new MockXLogService(oozieLogPath, oozieLogName); - OozieRollingPolicy orp = new OozieRollingPolicy() { - @Override - XLogService getXLogService() { - return xls; - } - - }; + OozieRollingPolicy orp = new OozieRollingPolicy(); + + if (calendarUnit == Calendar.DAY_OF_MONTH) { + orp.setFileNamePattern(oozieLogPath + "/" + oozieLogName + "-%d{yyyy-MM-dd}"); + } + else { + orp.setFileNamePattern(oozieLogPath + "/" + oozieLogName + "-%d{yyyy-MM-dd-HH}"); + + } orp.setMaxHistory(3); // only keep 3 newest logs Calendar cal = new GregorianCalendar(); final File f0 = new File(oozieLogPath, oozieLogName); f0.createNewFile(); f0.setLastModified(cal.getTimeInMillis()); - cal.add(Calendar.HOUR_OF_DAY, 1); - final File f1 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal) + ".gz"); + cal.add(calendarUnit, 1); + final File f1 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal, calendarUnit) + ".gz"); f1.createNewFile(); - cal.add(Calendar.HOUR_OF_DAY, 1); - final File f2 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal) + ".gz"); + cal.add(calendarUnit, 1); + final File f2 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal, calendarUnit) + ".gz"); f2.createNewFile(); - cal.add(Calendar.HOUR_OF_DAY, 1); - final File f3 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal) + ".gz"); + cal.add(calendarUnit, 1); + final File f3 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal, calendarUnit) + ".gz"); f3.createNewFile(); - cal.add(Calendar.HOUR_OF_DAY, 1); - final File f4 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal) + ".gz"); + cal.add(calendarUnit, 1); + final File f4 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal, calendarUnit) + ".gz"); f4.createNewFile(); // Test that it only deletes the oldest file (f1) @@ -87,13 +104,13 @@ public class TestOozieRollingPolicy extends XTestCase { }); assertTrue(f0.exists() && !f1.exists() && f2.exists() && f3.exists() && f4.exists()); - cal.add(Calendar.HOUR_OF_DAY, 1); - final File f5 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal)); + cal.add(calendarUnit, 1); + final File f5 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal, calendarUnit)); f5.createNewFile(); f5.setLastModified(cal.getTimeInMillis()); - cal.add(Calendar.HOUR_OF_DAY, -15); - final File f6 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal)); + cal.add(calendarUnit, -15); + final File f6 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal, calendarUnit)); f6.createNewFile(); f6.setLastModified(cal.getTimeInMillis()); @@ -111,9 +128,9 @@ public class TestOozieRollingPolicy extends XTestCase { f7.createNewFile(); f7.setLastModified(cal.getTimeInMillis()); - cal.add(Calendar.HOUR_OF_DAY, 1); - final File f8 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal)); - cal.add(Calendar.HOUR_OF_DAY, 15); + cal.add(calendarUnit, 1); + final File f8 = new File(oozieLogPath, oozieLogName + formatDateForFilename(cal, calendarUnit)); + cal.add(calendarUnit, 15); f8.createNewFile(); f8.setLastModified(cal.getTimeInMillis()); @@ -130,19 +147,22 @@ public class TestOozieRollingPolicy extends XTestCase { assertTrue(f0.exists() && !f1.exists() && !f2.exists() && !f3.exists() && f4.exists() && f5.exists() && !f6.exists() && f7.exists() && f8.exists()); } + - private String formatDateForFilename(Calendar cal) { + private String formatDateForFilename(Calendar cal, int calendarUnit) { int year = cal.get(Calendar.YEAR); int month = cal.get(Calendar.MONTH) + 1; int date = cal.get(Calendar.DATE); int hour = cal.get(Calendar.HOUR_OF_DAY); - + StringBuilder sb = new StringBuilder("-"); if (year < 10) { sb.append("000"); - } else if (year < 100) { + } + else if (year < 100) { sb.append("00"); - } else if (year < 1000) { + } + else if (year < 1000) { sb.append("0"); } sb.append(year); @@ -156,33 +176,15 @@ public class TestOozieRollingPolicy extends XTestCase { sb.append("0"); } sb.append(date); - sb.append("-"); - if (hour < 10) { - sb.append("0"); + if (calendarUnit == Calendar.HOUR_OF_DAY) { + sb.append("-"); + if (hour < 10) { + sb.append("0"); + } + sb.append(hour); + } - sb.append(hour); - return sb.toString(); } - - class MockXLogService extends XLogService { - - private String logPath; - private String logName; - - public MockXLogService(String lp, String ln) { - logPath = lp; - logName = ln; - } - - @Override - public String getOozieLogName() { - return logName; - } - @Override - public String getOozieLogPath() { - return logPath; - } - } } http://git-wip-us.apache.org/repos/asf/oozie/blob/e94a345a/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index e3925de..7e9c01b 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2312 oozie doesn't purge audit and error log (puru) OOZIE-2431 Support hive.metastore.uris as alias for hcat.metastore.uri (satishsaley via puru) OOZIE-1976 Specifying coordinator input datasets in more logical ways (puru) OOZIE-2444 Need conditional logic in bundles (satishsaley via puru)
