Repository: falcon Updated Branches: refs/heads/master ca7a91232 -> 7285a4487
FALCON-1372 Retention does not work in corner cases. Contributed by Balu Vellanki. Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/7285a448 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/7285a448 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/7285a448 Branch: refs/heads/master Commit: 7285a44875b6dae67610ff7a65fb6290c5aefbf5 Parents: ca7a912 Author: Ajay Yadav <[email protected]> Authored: Mon Nov 16 01:36:30 2015 +0530 Committer: Ajay Yadav <[email protected]> Committed: Mon Nov 16 01:36:30 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../org/apache/falcon/entity/FeedHelper.java | 10 ++++- .../falcon/entity/common/FeedDataPath.java | 8 ++-- .../apache/falcon/entity/FeedHelperTest.java | 43 ++++++++++++-------- 4 files changed, 43 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/7285a448/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 7ae9b0f..59d8f19 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -29,6 +29,8 @@ Trunk (Unreleased) OPTIMIZATIONS BUG FIXES + FALCON-1372 Retention does not work in corner cases(Balu Vellanki via Ajay Yadava) + FALCON-1569 Bug in setting the frequency of Feed retention coordinator(Sowmya Ramesh) FALCON-1578 post-processing action fails with javax.servlet.jsp.el.ELException(Venkat Ramachandran via Ajay Yadava) http://git-wip-us.apache.org/repos/asf/falcon/blob/7285a448/common/src/main/java/org/apache/falcon/entity/FeedHelper.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/FeedHelper.java b/common/src/main/java/org/apache/falcon/entity/FeedHelper.java index 8c55e41..d601c5d 100644 --- a/common/src/main/java/org/apache/falcon/entity/FeedHelper.java +++ b/common/src/main/java/org/apache/falcon/entity/FeedHelper.java @@ -475,8 +475,16 @@ public final class FeedHelper { //Reset other fields for (FeedDataPath.VARS var : FeedDataPath.VARS.values()) { if (!matchedVars.contains(var)) { - cal.set(var.getCalendarField(), 0); + switch (var.getCalendarField()) { + case Calendar.DAY_OF_MONTH: + cal.set(var.getCalendarField(), 1); + break; + default: + cal.set(var.getCalendarField(), 0); + } } + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); } return cal.getTime(); } http://git-wip-us.apache.org/repos/asf/falcon/blob/7285a448/common/src/main/java/org/apache/falcon/entity/common/FeedDataPath.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/common/FeedDataPath.java b/common/src/main/java/org/apache/falcon/entity/common/FeedDataPath.java index afe913d..51568fb 100644 --- a/common/src/main/java/org/apache/falcon/entity/common/FeedDataPath.java +++ b/common/src/main/java/org/apache/falcon/entity/common/FeedDataPath.java @@ -30,10 +30,12 @@ public final class FeedDataPath { /** * Standard variables for feed time components. */ - public static enum VARS { - YEAR("([0-9]{4})", Calendar.YEAR, 4), MONTH("(0[1-9]|1[0-2])", Calendar.MONTH, 2), + public enum VARS { + YEAR("([0-9]{4})", Calendar.YEAR, 4), + MONTH("(0[1-9]|1[0-2])", Calendar.MONTH, 2), DAY("(0[1-9]|1[0-9]|2[0-9]|3[0-1])", Calendar.DAY_OF_MONTH, 2), - HOUR("([0-1][0-9]|2[0-4])", Calendar.HOUR_OF_DAY, 2), MINUTE("([0-5][0-9]|60)", Calendar.MINUTE, 2); + HOUR("([0-1][0-9]|2[0-4])", Calendar.HOUR_OF_DAY, 2), + MINUTE("([0-5][0-9]|60)", Calendar.MINUTE, 2); private final Pattern pattern; private final String valuePattern; http://git-wip-us.apache.org/repos/asf/falcon/blob/7285a448/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java b/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java index 10dac49..4094556 100644 --- a/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java +++ b/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java @@ -30,23 +30,22 @@ import org.apache.falcon.entity.v0.cluster.Properties; import org.apache.falcon.entity.v0.cluster.Property; import org.apache.falcon.entity.v0.feed.Argument; import org.apache.falcon.entity.v0.feed.Arguments; -import org.apache.falcon.entity.v0.feed.ExtractMethod; -import org.apache.falcon.entity.v0.feed.Clusters; import org.apache.falcon.entity.v0.feed.ClusterType; +import org.apache.falcon.entity.v0.feed.Clusters; import org.apache.falcon.entity.v0.feed.Extract; +import org.apache.falcon.entity.v0.feed.ExtractMethod; import org.apache.falcon.entity.v0.feed.Feed; -import org.apache.falcon.entity.v0.feed.FieldsType; import org.apache.falcon.entity.v0.feed.FieldIncludeExclude; +import org.apache.falcon.entity.v0.feed.FieldsType; import org.apache.falcon.entity.v0.feed.Import; import org.apache.falcon.entity.v0.feed.Lifecycle; import org.apache.falcon.entity.v0.feed.Location; -import org.apache.falcon.entity.v0.feed.Locations; import org.apache.falcon.entity.v0.feed.LocationType; +import org.apache.falcon.entity.v0.feed.Locations; import org.apache.falcon.entity.v0.feed.MergeType; -import org.apache.falcon.entity.v0.feed.Source; import org.apache.falcon.entity.v0.feed.RetentionStage; +import org.apache.falcon.entity.v0.feed.Source; import org.apache.falcon.entity.v0.feed.Validity; - import org.apache.falcon.entity.v0.process.Input; import org.apache.falcon.entity.v0.process.Inputs; import org.apache.falcon.entity.v0.process.Output; @@ -290,21 +289,33 @@ public class FeedHelperTest extends AbstractTestBase { @DataProvider(name = "fsPathsforDate") public Object[][] createPathsForGetDate() { + final TimeZone utc = TimeZone.getTimeZone("UTC"); + final TimeZone pacificTime = TimeZone.getTimeZone("america/los_angeles"); + final TimeZone ist = TimeZone.getTimeZone("IST"); + return new Object[][] { - {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}", "/data/2015/01/01/00/30", "2015-01-01T00:30Z"}, - {"/data/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}", "/data/2015-01-01-01-00", "2015-01-01T01:00Z"}, - {"/data/${YEAR}/${MONTH}/${DAY}", "/data/2015/01/01", "2015-01-01T00:00Z"}, - {"/data/${YEAR}/${MONTH}/${DAY}/data", "/data/2015/01/01/data", "2015-01-01T00:00Z"}, - {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}", "/data/2015-01-01/00/30", null}, - {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}/data", "/data/2015-01-01/00/30", null}, - {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/data", "/data/2015/05/25/00/data/{p1}/p2", "2015-05-25T00:00Z"}, - {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}/data", "/data/2015/05/25/00/00/{p1}/p2", null}, + {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}", "/data/2015/01/01/00/30", utc, "2015-01-01T00:30Z"}, + {"/data/${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}", "/data/2015-01-01-01-00", utc, "2015-01-01T01:00Z"}, + {"/data/${YEAR}/${MONTH}/${DAY}", "/data/2015/01/01", utc, "2015-01-01T00:00Z"}, + {"/data/${YEAR}/${MONTH}/${DAY}/data", "/data/2015/01/01/data", utc, "2015-01-01T00:00Z"}, + {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}", "/data/2015-01-01/00/30", utc, null}, + {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}/data", "/data/2015-01-01/00/30", utc, null}, + {"/d/${YEAR}/${MONTH}/${DAY}/${HOUR}/data", "/d/2015/05/25/00/data/{p1}/p2", utc, "2015-05-25T00:00Z"}, + {"/data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}/data", "/data/2015/05/25/00/00/{p1}/p2", utc, null}, + {"/d/${YEAR}/${MONTH}/M", "/d/2015/11/M", utc, "2015-11-01T00:00Z"}, + {"/d/${YEAR}/${MONTH}/${DAY}/M", "/d/2015/11/02/M", utc, "2015-11-02T00:00Z"}, + {"/d/${YEAR}/${MONTH}/${DAY}/${HOUR}/M", "/d/2015/11/01/04/M", utc, "2015-11-01T04:00Z"}, + {"/d/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}/M", "/d/2015/11/01/04/15/M", utc, "2015-11-01T04:15Z"}, + {"/d/${YEAR}/${MONTH}/M", "/d/2015/11/M", pacificTime, "2015-11-01T07:00Z"}, + {"/d/${YEAR}/${MONTH}/${DAY}/M", "/d/2015/11/02/M", pacificTime, "2015-11-02T08:00Z"}, + {"/d/${YEAR}/${MONTH}/${DAY}/${HOUR}/M", "/d/2015/11/01/04/M", pacificTime, "2015-11-01T12:00Z"}, + {"/d/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}/M", "/d/2015/11/01/04/15/M", ist, "2015-10-31T22:45Z"}, }; } @Test(dataProvider = "fsPathsforDate") - public void testGetDateFromPath(String template, String path, String expectedDate) throws Exception { - Date date = FeedHelper.getDate(template, new Path(path), UTC); + public void testGetDateFromPath(String template, String path, TimeZone tz, String expectedDate) throws Exception { + Date date = FeedHelper.getDate(template, new Path(path), tz); Assert.assertEquals(SchemaHelper.formatDateUTC(date), expectedDate); }
