Repository: falcon Updated Branches: refs/heads/master 48d0723cc -> 8600e9465
FALCON-1437 Change DR recipes notification with Falcon notification. Contributed by Peeyush Bishnoi. Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/8600e946 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/8600e946 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/8600e946 Branch: refs/heads/master Commit: 8600e94654ea21b5cff1ea363560ffc17078aa53 Parents: 48d0723 Author: Sowmya Ramesh <[email protected]> Authored: Tue Sep 22 11:46:37 2015 -0700 Committer: Sowmya Ramesh <[email protected]> Committed: Tue Sep 22 11:46:37 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../resources/hdfs-replication-template.xml | 1 + .../resources/hdfs-replication-workflow.xml | 42 ---------------- .../main/resources/hdfs-replication.properties | 5 +- .../hive-disaster-recovery-secure-template.xml | 1 + .../hive-disaster-recovery-secure-workflow.xml | 50 +------------------- .../hive-disaster-recovery-secure.properties | 5 +- .../hive-disaster-recovery-template.xml | 1 + .../hive-disaster-recovery-workflow.xml | 50 +------------------- .../resources/hive-disaster-recovery.properties | 5 +- .../apache/falcon/recipe/RecipeToolOptions.java | 4 +- .../recipe/util/RecipeProcessBuilderUtils.java | 16 +++++++ .../resources/hdfs-replication-template.xml | 1 + .../test/resources/hdfs-replication.properties | 5 +- 14 files changed, 41 insertions(+), 147 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 79d7548..5a7c46e 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,8 @@ Trunk (Unreleased) FALCON-1401 MetadataMappingService fails to add an edge for a process instance(Pallavi Rao) NEW FEATURES + FALCON-1437 Change DR recipes notification with Falcon notification(Peeyush Bishnoi via Sowmya Ramesh) + FALCON-1027 Falcon proxy user support(Sowmya Ramesh) IMPROVEMENTS http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml ---------------------------------------------------------------------- diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml index de0f748..441a189 100644 --- a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml +++ b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-template.xml @@ -39,5 +39,6 @@ <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml" lib="##workflow.lib.path##"/> <retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##" attempts="3"/> + <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/> <ACL/> </process> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml ---------------------------------------------------------------------- diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml index e35773f..942421f 100644 --- a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml +++ b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml @@ -68,48 +68,6 @@ <arg>-availabilityFlag</arg> <arg>${availabilityFlag == 'NA' ? "NA" : availabilityFlag}</arg> </java> - <ok to="success"/> - <error to="failure"/> - </action> - <decision name="success"> - <switch> - <case to="successAlert"> - ${drNotificationReceivers ne 'NA'} - </case> - <default to="end"/> - </switch> - </decision> - <decision name="failure"> - <switch> - <case to="failureAlert"> - ${drNotificationReceivers ne 'NA'} - </case> - <default to="fail"/> - </switch> - </decision> - <action name="successAlert"> - <email xmlns="uri:oozie:email-action:0.2"> - <to>${drNotificationReceivers}</to> - <subject>INFO: HDFS DR workflow ${entityName} completed successfully</subject> - <body> - The HDFS DR workflow ${wf:id()} is successful. - Source = ${drSourceDir} - Target = ${drTargetClusterFS}${drTargetDir} - </body> - </email> - <ok to="end"/> - <error to="end"/> - </action> - <action name="failureAlert"> - <email xmlns="uri:oozie:email-action:0.2"> - <to>${drNotificationReceivers}</to> - <subject>ERROR: HDFS DR workflow ${entityName} failed</subject> - <body> - The workflow ${wf:id()} had issues and was killed. The error message is: ${wf:errorMessage(wf:lastErrorNode())} - Source = ${drSourceDir} - Target = ${drTargetClusterFS}${drTargetDir} - </body> - </email> <ok to="end"/> <error to="fail"/> </action> http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties ---------------------------------------------------------------------- diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties index 64ab6b8..7ef4095 100644 --- a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties +++ b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties @@ -73,5 +73,6 @@ distcpMaxMaps=1 # Change it to specify the bandwidth in MB for each mapper in DistCP distcpMapBandwidth=100 -##### Email on failure -drNotificationReceivers=NA +##### Email Notification for Falcon instance completion +falcon.recipe.notification.type=email +falcon.recipe.notification.receivers=NA \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml index 3afbef0..f0de091 100644 --- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml +++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-template.xml @@ -40,5 +40,6 @@ <workflow name="##workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/> <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/> + <notification type="##notification.type##" to="##notification.receivers##"/> <ACL/> </process> http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml index 7362c2e..74902b4 100644 --- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml +++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure-workflow.xml @@ -148,7 +148,7 @@ <arg>lastevents</arg> </java> <ok to="export-dr-replication"/> - <error to="failure"/> + <error to="fail"/> </action> <!-- Export Replication action --> <action name="export-dr-replication" cred="hive_src_credentials,hive2_src_credentials"> @@ -246,7 +246,7 @@ <arg>export</arg> </java> <ok to="import-dr-replication"/> - <error to="failure"/> + <error to="fail"/> </action> <!-- Import Replication action --> <action name="import-dr-replication" cred="hive_tgt_credentials,hive2_tgt_credentials"> @@ -343,52 +343,6 @@ <arg>-executionStage</arg> <arg>import</arg> </java> - <ok to="success"/> - <error to="failure"/> - </action> - <decision name="success"> - <switch> - <case to="successAlert"> - ${drNotificationReceivers ne 'NA'} - </case> - <default to="end"/> - </switch> - </decision> - <decision name="failure"> - <switch> - <case to="failureAlert"> - ${drNotificationReceivers ne 'NA'} - </case> - <default to="fail"/> - </switch> - </decision> - <action name="successAlert"> - <email xmlns="uri:oozie:email-action:0.2"> - <to>${drNotificationReceivers}</to> - <subject>INFO: Hive DR workflow ${drJobName} completed successfully</subject> - <body> - The Hive DR workflow ${wf:id()} is successful. - Source = ${sourceCluster} - Target = ${targetCluster} - DB Name = ${sourceDatabase} - Table Name = ${sourceTable} - </body> - </email> - <ok to="end"/> - <error to="end"/> - </action> - <action name="failureAlert"> - <email xmlns="uri:oozie:email-action:0.2"> - <to>${drNotificationReceivers}</to> - <subject>ERROR: Hive DR workflow ${drJobName} failed</subject> - <body> - The Hive DR workflow ${wf:id()} had issues and was killed. The error message is: ${wf:errorMessage(wf:lastErrorNode())} - Source = ${sourceCluster} - Target = ${targetCluster} - DB Name = ${sourceDatabase} - Table Name = ${sourceTable} - </body> - </email> <ok to="end"/> <error to="fail"/> </action> http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties index cc70ac4..df4a8fd 100644 --- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties +++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-secure.properties @@ -102,5 +102,6 @@ distcpMaxMaps=1 # Change it to specify the bandwidth in MB for each mapper in DistCP distcpMapBandwidth=100 -##### Email on failure -drNotificationReceivers=NA +##### Email Notification for Falcon instance completion +falcon.recipe.notification.type=email +falcon.recipe.notification.receivers=NA \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml index 3afbef0..f0de091 100644 --- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml +++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-template.xml @@ -40,5 +40,6 @@ <workflow name="##workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/> <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/> + <notification type="##notification.type##" to="##notification.receivers##"/> <ACL/> </process> http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml index c441998..72d40a3 100644 --- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml +++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery-workflow.xml @@ -84,7 +84,7 @@ <arg>lastevents</arg> </java> <ok to="export-dr-replication"/> - <error to="failure"/> + <error to="fail"/> </action> <!-- Export Replication action --> <action name="export-dr-replication"> @@ -160,7 +160,7 @@ <arg>export</arg> </java> <ok to="import-dr-replication"/> - <error to="failure"/> + <error to="fail"/> </action> <!-- Import Replication action --> <action name="import-dr-replication"> @@ -235,52 +235,6 @@ <arg>-executionStage</arg> <arg>import</arg> </java> - <ok to="success"/> - <error to="failure"/> - </action> - <decision name="success"> - <switch> - <case to="successAlert"> - ${drNotificationReceivers ne 'NA'} - </case> - <default to="end"/> - </switch> - </decision> - <decision name="failure"> - <switch> - <case to="failureAlert"> - ${drNotificationReceivers ne 'NA'} - </case> - <default to="fail"/> - </switch> - </decision> - <action name="successAlert"> - <email xmlns="uri:oozie:email-action:0.2"> - <to>${drNotificationReceivers}</to> - <subject>INFO: Hive DR workflow ${drJobName} completed successfully</subject> - <body> - The Hive DR workflow ${wf:id()} is successful. - Source = ${sourceCluster} - Target = ${targetCluster} - DB Name = ${sourceDatabase} - Table Name = ${sourceTable} - </body> - </email> - <ok to="end"/> - <error to="end"/> - </action> - <action name="failureAlert"> - <email xmlns="uri:oozie:email-action:0.2"> - <to>${drNotificationReceivers}</to> - <subject>ERROR: Hive DR workflow ${drJobName} failed</subject> - <body> - The Hive DR workflow ${wf:id()} had issues and was killed. The error message is: ${wf:errorMessage(wf:lastErrorNode())} - Source = ${sourceCluster} - Target = ${targetCluster} - DB Name = ${sourceDatabase} - Table Name = ${sourceTable} - </body> - </email> <ok to="end"/> <error to="fail"/> </action> http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties ---------------------------------------------------------------------- diff --git a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties index c2cbf5d..c2915fb 100644 --- a/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties +++ b/addons/recipes/hive-disaster-recovery/src/main/resources/hive-disaster-recovery.properties @@ -92,5 +92,6 @@ distcpMaxMaps=1 # Change it to specify the bandwidth in MB for each mapper in DistCP distcpMapBandwidth=100 -##### Email on failure -drNotificationReceivers=NA +##### Email Notification for Falcon instance completion +falcon.recipe.notification.type=email +falcon.recipe.notification.receivers=NA \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java b/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java index 5df9b0a..0ece90b 100644 --- a/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java +++ b/client/src/main/java/org/apache/falcon/recipe/RecipeToolOptions.java @@ -42,7 +42,9 @@ public enum RecipeToolOptions { RECIPE_ACL_OWNER("falcon.recipe.acl.owner", "Recipe acl owner", false), RECIPE_ACL_GROUP("falcon.recipe.acl.group", "Recipe acl group", false), RECIPE_ACL_PERMISSION("falcon.recipe.acl.permission", "Recipe acl permission", false), - RECIPE_NN_PRINCIPAL("falcon.recipe.nn.principal", "Recipe DFS NN principal", false); + RECIPE_NN_PRINCIPAL("falcon.recipe.nn.principal", "Recipe DFS NN principal", false), + RECIPE_NOTIFICATION_TYPE("falcon.recipe.notification.type", "Recipe Notification Type", false), + RECIPE_NOTIFICATION_ADDRESS("falcon.recipe.notification.receivers", "Recipe Email Notification receivers", false); private final String name; private final String description; http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java b/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java index 9522816..9477bb4 100644 --- a/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java +++ b/client/src/main/java/org/apache/falcon/recipe/util/RecipeProcessBuilderUtils.java @@ -26,6 +26,7 @@ import org.apache.falcon.entity.v0.Frequency; import org.apache.falcon.entity.v0.SchemaHelper; import org.apache.falcon.entity.v0.process.ACL; import org.apache.falcon.entity.v0.process.Cluster; +import org.apache.falcon.entity.v0.process.Notification; import org.apache.falcon.entity.v0.process.PolicyType; import org.apache.falcon.entity.v0.process.Property; import org.apache.falcon.entity.v0.process.Retry; @@ -111,6 +112,7 @@ public final class RecipeProcessBuilderUtils { bindWorkflowProperties(process.getWorkflow(), recipeProperties); bindRetryProperties(process.getRetry(), recipeProperties); + bindNotificationProperties(process.getNotification(), recipeProperties); bindACLProperties(process.getACL(), recipeProperties); bindTagsProperties(process, recipeProperties); bindCustomProperties(process.getProperties(), recipeProperties); @@ -175,6 +177,20 @@ public final class RecipeProcessBuilderUtils { } } + private static void bindNotificationProperties(final Notification processNotification, + final Properties recipeProperties) { + processNotification.setType(recipeProperties.getProperty( + RecipeToolOptions.RECIPE_NOTIFICATION_TYPE.getName())); + + String notificationAddress = recipeProperties.getProperty( + RecipeToolOptions.RECIPE_NOTIFICATION_ADDRESS.getName()); + if (StringUtils.isNotBlank(notificationAddress)) { + processNotification.setTo(notificationAddress); + } else { + processNotification.setTo("NA"); + } + } + private static void bindACLProperties(final ACL acl, final Properties recipeProperties) { String aclowner = recipeProperties.getProperty(RecipeToolOptions.RECIPE_ACL_OWNER.getName()); http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/webapp/src/test/resources/hdfs-replication-template.xml ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/hdfs-replication-template.xml b/webapp/src/test/resources/hdfs-replication-template.xml index b93cc0b..c038484 100644 --- a/webapp/src/test/resources/hdfs-replication-template.xml +++ b/webapp/src/test/resources/hdfs-replication-template.xml @@ -39,5 +39,6 @@ <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml" lib="##workflow.lib.path##"/> <retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##" attempts="3"/> + <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/> <ACL/> </process> http://git-wip-us.apache.org/repos/asf/falcon/blob/8600e946/webapp/src/test/resources/hdfs-replication.properties ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/hdfs-replication.properties b/webapp/src/test/resources/hdfs-replication.properties index 1b485b4..bd77a1c 100644 --- a/webapp/src/test/resources/hdfs-replication.properties +++ b/webapp/src/test/resources/hdfs-replication.properties @@ -41,5 +41,6 @@ distcpMaxMaps=1 # Change it to specify the bandwidth in MB for each mapper in DistCP distcpMapBandwidth=100 -##### Email on failure -drNotificationReceivers=NA \ No newline at end of file +##### Email Notification for Falcon instance completion #### +falcon.recipe.notification.type=email +falcon.recipe.notification.receivers=NA \ No newline at end of file
