Repository: flume Updated Branches: refs/heads/trunk 4eb2a3bb5 -> e6df16d78
FLUME-2889: Fixes to DateTime computations (Tristan Stevens via Roshan Naik) Project: http://git-wip-us.apache.org/repos/asf/flume/repo Commit: http://git-wip-us.apache.org/repos/asf/flume/commit/e6df16d7 Tree: http://git-wip-us.apache.org/repos/asf/flume/tree/e6df16d7 Diff: http://git-wip-us.apache.org/repos/asf/flume/diff/e6df16d7 Branch: refs/heads/trunk Commit: e6df16d782ae8917b443be81d6a5ad755e02f5c3 Parents: 4eb2a3b Author: Roshan Naik <[email protected]> Authored: Mon Apr 25 12:43:31 2016 -0700 Committer: Roshan Naik <[email protected]> Committed: Mon Apr 25 13:05:07 2016 -0700 ---------------------------------------------------------------------- .../org/apache/flume/source/SyslogParser.java | 11 +++- .../org/apache/flume/source/SyslogUtils.java | 40 ++++++++++++- .../apache/flume/source/TestSyslogUtils.java | 62 ++++++++++++++++++-- 3 files changed, 104 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flume/blob/e6df16d7/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java ---------------------------------------------------------------------- diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java b/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java index c8245ff..b57ffac 100644 --- a/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java +++ b/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java @@ -317,9 +317,16 @@ public class SyslogParser { return 0; } - // try to deal with boundary cases, i.e. new year's eve. // rfc3164 dates are really dumb. - // NB: cannot handle replaying of old logs or going back to the future + /* + * Some code to try and add some smarts to the year insertion as without a year in the message we + * need to make some educated guessing. + * First set the "fixed" to be the timestamp with the current year. + * If the "fixed" time is more than one month in the future then roll it back a year. + * If the "fixed" time is more than eleven months in the past then roll it forward a year. + * This gives us a 12 month rolling window (11 months in the past, 1 month in the future) of timestamps. + */ + if (date != null) { DateTime fixed = date.withYear(year); http://git-wip-us.apache.org/repos/asf/flume/blob/e6df16d7/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java ---------------------------------------------------------------------- diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java b/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java index 5a9f4c8..4866183 100644 --- a/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java +++ b/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java @@ -33,6 +33,7 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -369,7 +370,44 @@ public class SyslogUtils { // try the available time formats to timestamp for (int dt = 0; dt < fmt.dateFormat.size(); dt++) { try { - timeStamp = String.valueOf(fmt.dateFormat.get(dt).parse(value).getTime()); + Date parsedDate = fmt.dateFormat.get(dt).parse(value); + /* + * Some code to try and add some smarts to the year insertion. + * Original code just added the current year which was okay-ish, but around January 1st becomes + * pretty naïve. + * The current year is added above. This code, if the year has been added does the following: + * 1. Compute what the computed time, but one month in the past would be. + * 2. Compute what the computed time, but eleven months in the future would be. + * If the computed time is more than one month in the future then roll it back a year. + * If the computed time is more than eleven months in the past then roll it forward a year. + * This gives us a 12 month rolling window (11 months in the past, 1 month in the future) of timestamps. + */ + if (fmt.addYear) { + Calendar cal = Calendar.getInstance(); + cal.setTime(parsedDate); + Calendar calMinusOneMonth = Calendar.getInstance(); + calMinusOneMonth.setTime(parsedDate); + calMinusOneMonth.add(Calendar.MONTH, -1); + + Calendar calPlusElevenMonths = Calendar.getInstance(); + calPlusElevenMonths.setTime(parsedDate); + calPlusElevenMonths.add(Calendar.MONTH, +11); + + if (cal.getTimeInMillis() > System.currentTimeMillis() && calMinusOneMonth.getTimeInMillis() > System.currentTimeMillis()) { + //Need to roll back a year + Calendar c1 = Calendar.getInstance(); + c1.setTime(parsedDate); + c1.add(Calendar.YEAR, -1); + parsedDate = c1.getTime(); + } else if (cal.getTimeInMillis() < System.currentTimeMillis() && calPlusElevenMonths.getTimeInMillis() < System.currentTimeMillis() ) { + //Need to roll forward a year + Calendar c1 = Calendar.getInstance(); + c1.setTime(parsedDate); + c1.add(Calendar.YEAR, -1); + parsedDate = c1.getTime(); + } + } + timeStamp = String.valueOf(parsedDate.getTime()); break; // done. formatted the time } catch (ParseException e) { // Error formatting the timeStamp, try next format http://git-wip-us.apache.org/repos/asf/flume/blob/e6df16d7/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogUtils.java ---------------------------------------------------------------------- diff --git a/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogUtils.java b/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogUtils.java index be4598e..1c005ff 100644 --- a/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogUtils.java +++ b/flume-ng-core/src/test/java/org/apache/flume/source/TestSyslogUtils.java @@ -28,6 +28,7 @@ import org.junit.Test; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; +import java.util.Date; import java.util.HashSet; import java.util.Locale; import java.util.Map; @@ -130,25 +131,33 @@ public class TestSyslogUtils { @Test public void TestHeader9() throws ParseException { - String stamp1 = "Apr 11 13:14:04"; + SimpleDateFormat sdf = new SimpleDateFormat("MMM d hh:MM:ss"); + Calendar cal = Calendar.getInstance(); + + String year = String.valueOf(cal.get(Calendar.YEAR)); + String stamp1 = sdf.format(cal.getTime()); String format1 = "yyyyMMM d HH:mm:ss"; String host1 = "ubuntu-11.cloudera.com"; String data1 = "some msg"; // timestamp with 'Z' appended, translates to UTC String msg1 = "<10>" + stamp1 + " " + host1 + " " + data1 + "\n"; - checkHeader(msg1, String.valueOf(Calendar.getInstance().get(Calendar.YEAR)) + stamp1, + checkHeader(msg1, year + stamp1, format1, host1, data1); } @Test public void TestHeader10() throws ParseException { - String stamp1 = "Apr 1 13:14:04"; + SimpleDateFormat sdf = new SimpleDateFormat("MMM d hh:MM:ss"); + Calendar cal = Calendar.getInstance(); + + String year = String.valueOf(cal.get(Calendar.YEAR)); + String stamp1 = sdf.format(cal.getTime()); String format1 = "yyyyMMM d HH:mm:ss"; String host1 = "ubuntu-11.cloudera.com"; String data1 = "some msg"; // timestamp with 'Z' appended, translates to UTC String msg1 = "<10>" + stamp1 + " " + host1 + " " + data1 + "\n"; - checkHeader(msg1, String.valueOf(Calendar.getInstance().get(Calendar.YEAR)) + stamp1, + checkHeader(msg1, year + stamp1, format1, host1, data1); } @@ -169,15 +178,56 @@ public class TestSyslogUtils { @Test public void TestRfc3164HeaderApacheLogWithNulls() throws ParseException { - String stamp1 = "Apr 1 13:14:04"; + SimpleDateFormat sdf = new SimpleDateFormat("MMM d hh:MM:ss"); + Calendar cal = Calendar.getInstance(); + + String year = String.valueOf(cal.get(Calendar.YEAR)); + String stamp1 = sdf.format(cal.getTime()); String format1 = "yyyyMMM d HH:mm:ss"; String host1 = "ubuntu-11.cloudera.com"; String data1 = "- hyphen_null_breaks_5424_pattern [07/Jun/2012:14:46:44 -0600]"; String msg1 = "<10>" + stamp1 + " " + host1 + " " + data1 + "\n"; - checkHeader(msg1, String.valueOf(Calendar.getInstance().get(Calendar.YEAR)) + stamp1, + checkHeader(msg1, year + stamp1, format1, host1, data1); } + @Test + public void TestRfc3164Dates() throws ParseException { + /* + * This test creates a series of dates that range from 10 months in the past to (5 days short of) + * one month in the future. This tests that the year addition code is clever enough to handle scenarios + * where the event received was generated in a different year to what flume considers to be "current" + * (e.g. where there has been some lag somewhere, especially when flicking over on New Year's eve, or + * when you are about to flick over and the flume's system clock is slightly slower than the Syslog + * source's clock). + */ + for (int i=-10; i<=1; i++) { + SimpleDateFormat sdf = new SimpleDateFormat("MMM d hh:MM:ss"); + Date date = new Date(System.currentTimeMillis()); + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + cal.add(Calendar.MONTH, i); + + //Small tweak to avoid the 1 month in the future ticking over by a few seconds between now + //and when the checkHeader actually runs + if (i==1) { + cal.add(Calendar.DAY_OF_MONTH, -1); + } + + String stamp1 = sdf.format(cal.getTime()); + + String year = String.valueOf(cal.get(Calendar.YEAR)); + String format1 = "yyyyMMM d HH:mm:ss"; + String host1 = "ubuntu-11.cloudera.com"; + String data1 = "some msg"; + + // timestamp with 'Z' appended, translates to UTC + String msg1 = "<10>" + stamp1 + " " + host1 + " " + data1 + "\n"; + checkHeader(msg1, year + stamp1, + format1, host1, data1); + } + } + public static void checkHeader(String keepFields, String msg1, String stamp1, String format1, String host1, String data1) throws ParseException { SyslogUtils util;
