Repository: falcon Updated Branches: refs/heads/master aa18bfdae -> d97052f34
FALCON-1569 Bug in setting the frequency of Feed retention coordinator. Contributed by Sowmya Ramesh Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/d97052f3 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/d97052f3 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/d97052f3 Branch: refs/heads/master Commit: d97052f3423fae8213e4e3bf677ee21b632b82b0 Parents: aa18bfd Author: Sowmya Ramesh <[email protected]> Authored: Mon Nov 9 13:43:44 2015 -0800 Committer: Sowmya Ramesh <[email protected]> Committed: Mon Nov 9 13:43:44 2015 -0800 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../feed/FeedRetentionCoordinatorBuilder.java | 8 ++-- .../feed/OozieFeedWorkflowBuilderTest.java | 39 ++++++++++++++++++++ .../apache/falcon/unit/FalconUnitTestBase.java | 6 ++- 4 files changed, 50 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/d97052f3/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 586847e..5a964da 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -27,6 +27,8 @@ Trunk (Unreleased) OPTIMIZATIONS BUG FIXES + FALCON-1569 Bug in setting the frequency of Feed retention coordinator(Sowmya Ramesh) + FALCON-1578 post-processing action fails with javax.servlet.jsp.el.ELException(Venkat Ramachandran via Ajay Yadava) FALCON-1579 post-processing action fails with javax.servlet.jsp.el.ELException (Sowmya Ramesh via Pallavi Rao) http://git-wip-us.apache.org/repos/asf/falcon/blob/d97052f3/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java index 0809bb6..69ca2c3 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java @@ -24,7 +24,7 @@ import org.apache.falcon.LifeCycle; import org.apache.falcon.Tag; import org.apache.falcon.entity.EntityUtil; import org.apache.falcon.entity.FeedHelper; -import org.apache.falcon.entity.v0.Frequency.TimeUnit; +import org.apache.falcon.entity.v0.Frequency; import org.apache.falcon.entity.v0.SchemaHelper; import org.apache.falcon.entity.v0.cluster.Cluster; import org.apache.falcon.entity.v0.feed.Feed; @@ -34,6 +34,7 @@ import org.apache.falcon.oozie.OozieOrchestrationWorkflowBuilder; import org.apache.falcon.oozie.coordinator.ACTION; import org.apache.falcon.oozie.coordinator.COORDINATORAPP; import org.apache.falcon.oozie.coordinator.WORKFLOW; +import org.apache.falcon.util.DateUtil; import org.apache.hadoop.fs.Path; import java.util.Arrays; @@ -67,8 +68,9 @@ public class FeedRetentionCoordinatorBuilder extends OozieCoordinatorBuilder<Fee coord.setStart(SchemaHelper.formatDateUTC(new Date())); } coord.setTimezone(entity.getTimezone().getID()); - TimeUnit timeUnit = entity.getFrequency().getTimeUnit(); - if (timeUnit == TimeUnit.hours || timeUnit == TimeUnit.minutes) { + Frequency entityFrequency = entity.getFrequency(); + Frequency defaultFrequency = new Frequency("hours(24)"); + if (DateUtil.getFrequencyInMillis(entityFrequency) < DateUtil.getFrequencyInMillis(defaultFrequency)) { coord.setFrequency("${coord:hours(6)}"); } else { coord.setFrequency("${coord:days(1)}"); http://git-wip-us.apache.org/repos/asf/falcon/blob/d97052f3/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java ---------------------------------------------------------------------- diff --git a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java index 5e93027..176a15e 100644 --- a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java +++ b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java @@ -29,6 +29,7 @@ import org.apache.falcon.entity.Storage; import org.apache.falcon.entity.store.ConfigurationStore; import org.apache.falcon.entity.v0.Entity; import org.apache.falcon.entity.v0.EntityType; +import org.apache.falcon.entity.v0.Frequency; import org.apache.falcon.entity.v0.SchemaHelper; import org.apache.falcon.entity.v0.cluster.Cluster; import org.apache.falcon.entity.v0.cluster.ClusterLocationType; @@ -187,6 +188,44 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase { } @Test + public void testRetentionFrequency() throws Exception { + feed.setFrequency(new Frequency("minutes(36000)")); + buildCoordAndValidateFrequency("${coord:days(1)}"); + + feed.setFrequency(new Frequency("hours(2)")); + buildCoordAndValidateFrequency("${coord:hours(6)}"); + + feed.setFrequency(new Frequency("minutes(50)")); + buildCoordAndValidateFrequency("${coord:hours(6)}"); + + feed.setFrequency(new Frequency("days(1)")); + buildCoordAndValidateFrequency("${coord:days(1)}"); + } + + private void buildCoordAndValidateFrequency(final String expectedFrequency) throws Exception { + // retention on src cluster + OozieCoordinatorBuilder builder = OozieCoordinatorBuilder.get(feed, Tag.RETENTION); + List<Properties> srcCoords = builder.buildCoords( + srcCluster, new Path("/projects/falcon/")); + COORDINATORAPP srcCoord = getCoordinator(srcMiniDFS, srcCoords.get(0).getProperty(OozieEntityBuilder + .ENTITY_PATH)); + + // Assert src coord frequency + Assert.assertEquals(srcCoord.getFrequency(), expectedFrequency); + + // retention on target cluster + OozieEntityBuilder entityBuilder = OozieEntityBuilder.get(feed); + Path bundlePath = new Path("/projects/falcon/"); + entityBuilder.build(trgCluster, bundlePath); + BUNDLEAPP bundle = getBundle(trgMiniDFS.getFileSystem(), bundlePath); + List<COORDINATOR> coords = bundle.getCoordinator(); + + COORDINATORAPP tgtCoord = getCoordinator(trgMiniDFS, coords.get(0).getAppPath()); + // Assert target coord frequency + Assert.assertEquals(tgtCoord.getFrequency(), expectedFrequency); + } + + @Test public void testReplicationCoordsForFSStorage() throws Exception { OozieEntityBuilder builder = OozieEntityBuilder.get(feed); Path bundlePath = new Path("/projects/falcon/"); http://git-wip-us.apache.org/repos/asf/falcon/blob/d97052f3/unit/src/test/java/org/apache/falcon/unit/FalconUnitTestBase.java ---------------------------------------------------------------------- diff --git a/unit/src/test/java/org/apache/falcon/unit/FalconUnitTestBase.java b/unit/src/test/java/org/apache/falcon/unit/FalconUnitTestBase.java index bd81798..af5a3c5 100644 --- a/unit/src/test/java/org/apache/falcon/unit/FalconUnitTestBase.java +++ b/unit/src/test/java/org/apache/falcon/unit/FalconUnitTestBase.java @@ -350,11 +350,13 @@ public class FalconUnitTestBase { public InstancesResult.WorkflowStatus getRetentionStatus(String feedName, String cluster) throws FalconException, FalconCLIException { Feed feedEntity = EntityUtil.getEntity(EntityType.FEED, feedName); - Frequency.TimeUnit timeUnit = feedEntity.getFrequency().getTimeUnit(); + + Frequency feedFrequency = feedEntity.getFrequency(); + Frequency defaultFrequency = new Frequency("hours(24)"); long endTimeInMillis = System.currentTimeMillis() + 30000; String endTime = DateUtil.getDateFormatFromTime(endTimeInMillis); long startTimeInMillis; - if (timeUnit == Frequency.TimeUnit.hours || timeUnit == Frequency.TimeUnit.minutes) { + if (DateUtil.getFrequencyInMillis(feedFrequency) < DateUtil.getFrequencyInMillis(defaultFrequency)) { startTimeInMillis = endTimeInMillis - (6 * DateUtil.HOUR_IN_MILLIS); } else { startTimeInMillis = endTimeInMillis - (24 * DateUtil.HOUR_IN_MILLIS);
