Repository: falcon Updated Branches: refs/heads/master 8e0d5c584 -> 4783a19c4
FALCON-1499 Lifecycle Retention Issues. Contributed by Ajay Yadava. Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/4783a19c Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/4783a19c Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/4783a19c Branch: refs/heads/master Commit: 4783a19c4cfbb673a647b4382961fb0df9b034a6 Parents: 8e0d5c5 Author: Ajay Yadava <[email protected]> Authored: Thu Oct 1 12:28:34 2015 +0530 Committer: Ajay Yadava <[email protected]> Committed: Thu Oct 1 13:48:55 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ common/src/main/resources/startup.properties | 4 +++- .../engine/oozie/retention/AgeBasedCoordinatorBuilder.java | 2 ++ .../engine/oozie/retention/AgeBasedWorkflowBuilder.java | 9 +++++++-- src/conf/startup.properties | 4 +++- 5 files changed, 17 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/4783a19c/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 22aa5ff..b2104f4 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -47,6 +47,8 @@ Trunk (Unreleased) FALCON-1403 Revisit IT cleanup and teardown(Narayan Periwal via Pallavi Rao) BUG FIXES + FALCON-1499 Lifecycle Retention Issues(Ajay Yadava) + FALCON-1469 Flaky EntityManagerJerseyIT#testDuplicateSubmitCommands(Sandeep Samudrala via Ajay Yadava) FALCON-1470 HiveDRStatusStoreTest should fail when using fakeGroup to create StatusStore(Balu Vellanki via Sowmya Ramesh) http://git-wip-us.apache.org/repos/asf/falcon/blob/4783a19c/common/src/main/resources/startup.properties ---------------------------------------------------------------------- diff --git a/common/src/main/resources/startup.properties b/common/src/main/resources/startup.properties index 3383129..e853c5a 100644 --- a/common/src/main/resources/startup.properties +++ b/common/src/main/resources/startup.properties @@ -34,6 +34,8 @@ *.application.services=org.apache.falcon.security.AuthenticationInitializationService,\ org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ org.apache.falcon.service.ProcessSubscriberService,\ + org.apache.falcon.service.FeedSLAMonitoringService,\ + org.apache.falcon.service.LifecyclePolicyMap,\ org.apache.falcon.entity.store.ConfigurationStore,\ org.apache.falcon.rerun.service.RetryService,\ org.apache.falcon.rerun.service.LateRunService,\ @@ -76,7 +78,7 @@ #Configurations used in UTs debug.config.store.uri=file://${user.dir}/target/store #Location to store state of Feed SLA monitoring service -debug.pending.feed.instances.store.uri = file://${user.dir}/data/sla/pendingfeedinstances +debug.feed.sla.service.store.uri= file://${user.dir}/target/data/sla/pendingfeedinstances debug.config.oozie.conf.uri=${user.dir}/target/oozie debug.system.lib.location=${system.lib.location} debug.broker.url=vm://localhost http://git-wip-us.apache.org/repos/asf/falcon/blob/4783a19c/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java ---------------------------------------------------------------------- diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java index 4601070..e3e7fec 100644 --- a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java +++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedCoordinatorBuilder.java @@ -83,6 +83,8 @@ public final class AgeBasedCoordinatorBuilder { CONTROLS controls = new CONTROLS(); controls.setExecution(ExecutionType.LAST_ONLY.value()); controls.setTimeout(String.valueOf(frequencyInMillis / (1000 * 60))); + controls.setConcurrency("1"); + controls.setThrottle("1"); coord.setControls(controls); coord.setFrequency("${coord:" + retentionFrequency.toString() + "}"); http://git-wip-us.apache.org/repos/asf/falcon/blob/4783a19c/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedWorkflowBuilder.java ---------------------------------------------------------------------- diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedWorkflowBuilder.java b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedWorkflowBuilder.java index 0a87213..34b186e 100644 --- a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedWorkflowBuilder.java +++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/retention/AgeBasedWorkflowBuilder.java @@ -18,6 +18,7 @@ package org.apache.falcon.lifecycle.engine.oozie.retention; +import org.apache.commons.lang3.StringUtils; import org.apache.falcon.FalconException; import org.apache.falcon.LifeCycle; import org.apache.falcon.Tag; @@ -82,8 +83,12 @@ public final class AgeBasedWorkflowBuilder { props.putAll(FeedHelper.getUserWorkflowProperties(LifeCycle.EVICTION)); // override the queueName and priority RetentionStage retentionStage = FeedHelper.getRetentionStage(feed, cluster.getName()); - props.put(OozieBuilderUtils.MR_QUEUE_NAME, retentionStage.getQueue()); - props.put(OozieBuilderUtils.MR_JOB_PRIORITY, retentionStage.getPriority()); + if (StringUtils.isNotBlank(retentionStage.getQueue())) { + props.put(OozieBuilderUtils.MR_QUEUE_NAME, retentionStage.getQueue()); + } + if (StringUtils.isNotBlank(retentionStage.getPriority())) { + props.put(OozieBuilderUtils.MR_JOB_PRIORITY, retentionStage.getPriority()); + } if (EntityUtil.isTableStorageType(cluster, feed)) { setupHiveCredentials(cluster, buildPath, workflow); http://git-wip-us.apache.org/repos/asf/falcon/blob/4783a19c/src/conf/startup.properties ---------------------------------------------------------------------- diff --git a/src/conf/startup.properties b/src/conf/startup.properties index 8e4ce97..37cb044 100644 --- a/src/conf/startup.properties +++ b/src/conf/startup.properties @@ -39,6 +39,7 @@ org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ org.apache.falcon.service.ProcessSubscriberService,\ org.apache.falcon.service.FeedSLAMonitoringService,\ + org.apache.falcon.service.LifecyclePolicyMap,\ org.apache.falcon.entity.store.ConfigurationStore,\ org.apache.falcon.rerun.service.RetryService,\ org.apache.falcon.rerun.service.LateRunService,\ @@ -48,7 +49,8 @@ org.apache.falcon.service.ProxyUserService ##### Prism Services ##### -prism.application.services=org.apache.falcon.entity.store.ConfigurationStore +prism.application.services=org.apache.falcon.service.LifecyclePolicyMap,\ + org.apache.falcon.entity.store.ConfigurationStore # List of Lifecycle policies configured.
