Repository: falcon Updated Branches: refs/heads/master 7354f870b -> 3a7c993b5
FALCON-2116 Queue defined in <retention-stage> is not getting used in actual retention job Author: sandeep <[email protected]> Reviewers: @pallavi-rao Closes #266 from sandeepSamudrala/FALCON-2116 and squashes the following commits: a23fe90 [sandeep] FALCON-2116 Queue defined in <retention-stage> is not getting used in actual retention job d6dc8bf [sandeep] Merge branch 'master' of https://github.com/apache/falcon 1bb8d3c [sandeep] Merge branch 'master' of https://github.com/apache/falcon c065566 [sandeep] reverting last line changes made 1a4dcd2 [sandeep] rebased and resolved the conflicts from master 271318b [sandeep] FALCON-2097. Adding UT to the new method for getting next instance time with Delay. a94d4fe [sandeep] rebasing from master 9e68a57 [sandeep] FALCON-298. Feed update with replication delay creates holes Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/3a7c993b Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/3a7c993b Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/3a7c993b Branch: refs/heads/master Commit: 3a7c993b55564d11ae79d25d32fd5fbd63ce4fe9 Parents: 7354f87 Author: sandeep <[email protected]> Authored: Wed Aug 17 09:30:13 2016 +0530 Committer: Pallavi Rao <[email protected]> Committed: Wed Aug 17 09:30:13 2016 +0530 ---------------------------------------------------------------------- .../engine/oozie/retention/AgeBasedCoordinatorBuilder.java | 8 ++++++++ .../falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java | 3 ++- .../src/test/resources/feed/fs-retention-lifecycle-feed.xml | 3 ++- 3 files changed, 12 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/3a7c993b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java ---------------------------------------------------------------------- diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java index 7a25d86..f7b31e0 100644 --- a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java +++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java @@ -41,6 +41,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Date; +import java.util.List; import java.util.Properties; /** @@ -100,6 +101,13 @@ public final class AgeBasedCoordinatorBuilder { Path buildPath = OozieBuilderUtils.getBuildPath(basePath, LifeCycle.EVICTION.getTag()); Properties props = OozieBuilderUtils.createCoordDefaultConfiguration(coordName, feed); props.putAll(OozieBuilderUtils.getProperties(buildPath, coordName)); + props.putAll(EntityUtil.getEntityProperties(feed)); + props.put("queueName", FeedHelper.getLifecycleRetentionQueue(feed, cluster.getName())); + List<org.apache.falcon.entity.v0.feed.Property> retentionProperties = + FeedHelper.getLifecycle(feed, cluster.getName()).getRetentionStage().getProperties().getProperties(); + for (org.apache.falcon.entity.v0.feed.Property retentionProperty : retentionProperties) { + props.put(retentionProperty.getName(), retentionProperty.getValue()); + } WORKFLOW workflow = new WORKFLOW(); String entityPath = wfProp.getProperty(OozieBuilderUtils.ENTITY_PATH); http://git-wip-us.apache.org/repos/asf/falcon/blob/3a7c993b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java ---------------------------------------------------------------------- diff --git a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java index fde5532..08d3339 100644 --- a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java +++ b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java @@ -189,6 +189,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase { assertLibExtensions(coord, "retention"); HashMap<String, String> props = getCoordProperties(coord); Assert.assertEquals(props.get("ENTITY_PATH"), bundlePath.toString() + "/RETENTION"); + Assert.assertEquals(props.get("queueName"), "ageBasedDeleteQueue"); Assert.assertEquals(coord.getFrequency(), "${coord:hours(17)}"); Assert.assertEquals(coord.getEnd(), endTime); Assert.assertEquals(coord.getTimezone(), "UTC"); @@ -197,7 +198,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase { Assert.assertEquals(wfProps.get("feedNames"), lifecycleRetentionFeed.getName()); Assert.assertTrue(StringUtils.equals(wfProps.get("entityType"), EntityType.FEED.name())); Assert.assertEquals(wfProps.get("userWorkflowEngine"), "falcon"); - Assert.assertEquals(wfProps.get("queueName"), "retention"); + Assert.assertEquals(wfProps.get("queueName"), "ageBasedDeleteQueue"); Assert.assertEquals(wfProps.get("limit"), "hours(2)"); Assert.assertEquals(wfProps.get("jobPriority"), "LOW"); } http://git-wip-us.apache.org/repos/asf/falcon/blob/3a7c993b/oozie/src/test/resources/feed/fs-retention-lifecycle-feed.xml ---------------------------------------------------------------------- diff --git a/oozie/src/test/resources/feed/fs-retention-lifecycle-feed.xml b/oozie/src/test/resources/feed/fs-retention-lifecycle-feed.xml index 2cadfe0..a230f15 100644 --- a/oozie/src/test/resources/feed/fs-retention-lifecycle-feed.xml +++ b/oozie/src/test/resources/feed/fs-retention-lifecycle-feed.xml @@ -45,12 +45,13 @@ <properties> <property name="maxMaps" value="33" /> <property name="mapBandwidth" value="2" /> + <property name="queueName" value="retentionQueue" /> </properties> <lifecycle> <retention-stage> <frequency>hours(17)</frequency> - <queue>retention</queue> + <queue>ageBasedDeleteQueue</queue> <priority>LOW</priority> <properties> <property name="retention.policy.agebaseddelete.limit" value="hours(2)"></property>
