Repository: oozie Updated Branches: refs/heads/master 7f5398830 -> 2dfaa4a74
OOZIE-1737 Oozie log streaming is slow (puru via rohini) - amend patch Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/2dfaa4a7 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/2dfaa4a7 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/2dfaa4a7 Branch: refs/heads/master Commit: 2dfaa4a74f9336cf27769483783760950bca0b94 Parents: 7f53988 Author: Rohini Palaniswamy <[email protected]> Authored: Fri May 16 09:08:00 2014 -0700 Committer: Rohini Palaniswamy <[email protected]> Committed: Fri May 16 09:08:00 2014 -0700 ---------------------------------------------------------------------- .../src/main/java/org/apache/oozie/util/XLogFilter.java | 10 ++++++++++ .../java/org/apache/oozie/util/XLogUserFilterParam.java | 12 +++++------- .../java/org/apache/oozie/util/TestLogStreamer.java | 8 ++++---- 3 files changed, 19 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/2dfaa4a7/core/src/main/java/org/apache/oozie/util/XLogFilter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/XLogFilter.java b/core/src/main/java/org/apache/oozie/util/XLogFilter.java index 0007c76..a1a7fbc 100644 --- a/core/src/main/java/org/apache/oozie/util/XLogFilter.java +++ b/core/src/main/java/org/apache/oozie/util/XLogFilter.java @@ -52,6 +52,7 @@ public class XLogFilter { private Date startDate; private boolean isActionList = false; private String formattedEndDate; + private String formattedStartDate; // TODO Patterns to be read from config file private static final String DEFAULT_REGEX = "[^\\]]*"; @@ -121,6 +122,11 @@ public class XLogFilter { * @return */ public boolean matches(ArrayList<String> logParts) { + if (getStartDate() != null) { + if (logParts.get(0).substring(0, 19).compareTo(getFormattedStartDate()) < 0) { + return false; + } + } String logLevel = logParts.get(1); String logMessage = logParts.get(2); if (this.logLevels == null || this.logLevels.containsKey(logLevel.toUpperCase())) { @@ -206,6 +212,9 @@ public class XLogFilter { return formattedEndDate; } + public String getFormattedStartDate() { + return formattedStartDate; + } public Date getStartDate() { return startDate; @@ -274,6 +283,7 @@ public class XLogFilter { } formattedEndDate = XLogUserFilterParam.dt.get().format(getEndDate()); + formattedStartDate = XLogUserFilterParam.dt.get().format(getStartDate()); } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/2dfaa4a7/core/src/main/java/org/apache/oozie/util/XLogUserFilterParam.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/XLogUserFilterParam.java b/core/src/main/java/org/apache/oozie/util/XLogUserFilterParam.java index 6e49e11..e3ed55d 100644 --- a/core/src/main/java/org/apache/oozie/util/XLogUserFilterParam.java +++ b/core/src/main/java/org/apache/oozie/util/XLogUserFilterParam.java @@ -56,6 +56,8 @@ public class XLogUserFilterParam { private boolean isDebug = false; private String searchText; + private String params; + public static final ThreadLocal<SimpleDateFormat> dt = new ThreadLocal<SimpleDateFormat>() { @Override protected SimpleDateFormat initialValue() { @@ -104,6 +106,8 @@ public class XLogUserFilterParam { * @throws Exception */ private void parseFilterParam(String param) throws Exception { + this.params = param; + if (StringUtils.isEmpty(param)) { return; } @@ -264,13 +268,7 @@ public class XLogUserFilterParam { @Override public String toString() { - StringBuffer sb = new StringBuffer(); - sb.append(START_TIME).append("=").append(getStartDate()).append(";"); - sb.append(END_TIME).append("=").append(getEndDate()).append(";"); - sb.append(LOG_LEVEL).append("=").append(getLogLevel()).append(";"); - sb.append(LIMIT).append("=").append(getLimit()).append(";"); - sb.append(RECENT_LOG_OFFSET).append("=").append(getRecent()).append(";"); - return sb.toString(); + return params; } private int getOffsetInMinute(String offset) throws IOException { http://git-wip-us.apache.org/repos/asf/oozie/blob/2dfaa4a7/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java b/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java index bc19341..6f983a7 100644 --- a/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java +++ b/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java @@ -22,6 +22,7 @@ import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; import java.io.StringWriter; +import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; @@ -30,7 +31,6 @@ import java.util.zip.GZIPOutputStream; import org.apache.oozie.command.CommandException; import org.apache.oozie.service.ServiceException; import org.apache.oozie.service.Services; -import org.apache.oozie.service.XLogService; import org.apache.oozie.test.XTestCase; public class TestLogStreamer extends XTestCase { @@ -40,7 +40,7 @@ public class TestLogStreamer extends XTestCase { private final static SimpleDateFormat filenameDateFormatter = new SimpleDateFormat("yyyy-MM-dd-HH"); - public void testStreamLog() throws IOException, CommandException, ServiceException { + public void testStreamLog() throws IOException, CommandException, ServiceException, ParseException { new Services().init(); long currTime = System.currentTimeMillis(); @@ -148,7 +148,7 @@ public class TestLogStreamer extends XTestCase { xf.setLogLevel("DEBUG|INFO"); xf.setParameter("JOB", "14-200904160239--example-forkjoinwf"); XLogStreamer str = new XLogStreamer(xf, getTestCaseDir(), "oozie.log", 1); - str.streamLog(sw, new Date(currTime - 10 * 3600000), new Date(currTime - 5 * 3600000), 4096); + str.streamLog(sw, DateUtils.parseDateOozieTZ("2009-06-24T02:43Z"), new Date(currTime - 5 * 3600000), 4096); String[] out = sw.toString().split("\n"); // Check if the retrieved log content is of length seven lines after filtering based on time window, file name // pattern and parameters like JobId, Username etc. and/or based on log level like INFO, DEBUG, etc. @@ -294,7 +294,7 @@ public class TestLogStreamer extends XTestCase { xf.setLogLevel("DEBUG|INFO"); XLogStreamer str = new XLogStreamer(xf, getTestCaseDir(), "oozie.log", 1); - str.streamLog(sw, new Date(currTime - 5000), new Date(currTime + 5000), 4096); + str.streamLog(sw, null, null, 4096); String[] out = sw.toString().split("\n"); // Check if the retrieved log content is of length five lines after filtering; we expect the first five lines because the // filtering shouldn't care whether or not there is a dash while the last five lines don't pass the normal filtering
