Repository: falcon Updated Branches: refs/heads/master 89040a296 -> 44d5028f6
FALCON-1560 Lifecycle does not allow feed with frequency greater than days(1). Contributed by Ajay Yadava. Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/44d5028f Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/44d5028f Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/44d5028f Branch: refs/heads/master Commit: 44d5028f6a9e0994e6658611fcf9d9b17e5d39a6 Parents: 89040a2 Author: Ajay Yadav <[email protected]> Authored: Thu Oct 29 11:01:19 2015 +0530 Committer: Ajay Yadav <[email protected]> Committed: Thu Oct 29 11:01:19 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../java/org/apache/falcon/entity/FeedHelper.java | 10 ++++++---- .../org/apache/falcon/entity/FeedHelperTest.java | 11 +++++++++-- .../falcon/entity/parser/FeedEntityParserTest.java | 16 ++++++++++++++++ 4 files changed, 33 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/44d5028f/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index b5980be..9305340 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -93,6 +93,8 @@ Branch : 0.8 (Proposed Release version: 0.8) FALCON-1403 Revisit IT cleanup and teardown(Narayan Periwal via Pallavi Rao) BUG FIXES + FALCON-1560 Lifecycle does not allow feed with frequency greater than days(1) (Ajay Yadava). + FALCON-1556 Falcon build fails when building with hivedr profile(Sowmya Ramesh) FALCON-1558 Sla API shows pending instance although _SUCCESS directory is present(Pragya Mittal via Ajay Yadava). http://git-wip-us.apache.org/repos/asf/falcon/blob/44d5028f/common/src/main/java/org/apache/falcon/entity/FeedHelper.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/FeedHelper.java b/common/src/main/java/org/apache/falcon/entity/FeedHelper.java index 2c65eba..0b2e94f 100644 --- a/common/src/main/java/org/apache/falcon/entity/FeedHelper.java +++ b/common/src/main/java/org/apache/falcon/entity/FeedHelper.java @@ -50,6 +50,7 @@ import org.apache.falcon.resource.EntityList; import org.apache.falcon.resource.FeedInstanceResult; import org.apache.falcon.resource.SchedulableEntityInstance; import org.apache.falcon.util.BuildProperties; +import org.apache.falcon.util.DateUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; @@ -986,11 +987,12 @@ public final class FeedHelper { if (retentionStage != null && retentionStage.getFrequency() != null) { retentionFrequency = retentionStage.getFrequency(); } else { - Frequency.TimeUnit timeUnit = feed.getFrequency().getTimeUnit(); - if (timeUnit == Frequency.TimeUnit.hours || timeUnit == Frequency.TimeUnit.minutes) { - retentionFrequency = new Frequency("hours(6)"); + Frequency feedFrequency = feed.getFrequency(); + Frequency defaultFrequency = new Frequency("hours(6)"); + if (DateUtil.getFrequencyInMillis(feedFrequency) < DateUtil.getFrequencyInMillis(defaultFrequency)) { + retentionFrequency = defaultFrequency; } else { - retentionFrequency = new Frequency("days(1)"); + retentionFrequency = new Frequency(feedFrequency.toString()); } } return retentionFrequency; http://git-wip-us.apache.org/repos/asf/falcon/blob/44d5028f/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java b/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java index 60bf1be..10dac49 100644 --- a/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java +++ b/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java @@ -832,15 +832,22 @@ public class FeedHelperTest extends AbstractTestBase { cluster.setName("cluster1"); clusters.getClusters().add(cluster); feed.setClusters(clusters); - Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new Frequency("days(1)")); + Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new Frequency("days(10)")); feed.setFrequency(new Frequency("hours(1)")); Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new Frequency("hours(6)")); - feed.setFrequency(new Frequency("minutes(5)")); + feed.setFrequency(new Frequency("minutes(10)")); Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new Frequency("hours(6)")); + feed.setFrequency(new Frequency("hours(7)")); + Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new Frequency("hours(7)")); + + feed.setFrequency(new Frequency("days(2)")); + Assert.assertEquals(FeedHelper.getRetentionFrequency(feed, cluster.getName()), new Frequency("days(2)")); + // lifecycle at both level - retention only at global level. + feed.setFrequency(new Frequency("hours(1)")); Lifecycle globalLifecycle = new Lifecycle(); RetentionStage globalRetentionStage = new RetentionStage(); globalRetentionStage.setFrequency(new Frequency("hours(2)")); http://git-wip-us.apache.org/repos/asf/falcon/blob/44d5028f/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java index 7f4abce..a583f52 100644 --- a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java +++ b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java @@ -208,6 +208,22 @@ public class FeedEntityParserTest extends AbstractTestBase { parser.validate(feed); // no validation exception should be thrown } + @Test + public void testDefaultRetentionFrequencyConflict() throws Exception { + Feed feed = parser.parseAndValidate(this.getClass().getResourceAsStream(FEED3_XML)); + feed.getLifecycle().getRetentionStage().setFrequency(null); + feed.getClusters().getClusters().get(0).getLifecycle().getRetentionStage().setFrequency(null); + feed.setFrequency(Frequency.fromString("minutes(10)")); + parser.validate(feed); // shouldn't throw a validation exception + + + feed.setFrequency(Frequency.fromString("hours(7)")); + parser.validate(feed); // shouldn't throw a validation exception + + feed.setFrequency(Frequency.fromString("days(2)")); + parser.validate(feed); // shouldn't throw a validation exception + } + @Test(expectedExceptions = ValidationException.class, expectedExceptionsMessageRegExp = ".*Retention can not be more frequent than data availability.*") public void testRetentionFrequentThanFeed() throws Exception {
