Joal has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/335067 )
Change subject: Update oozie pageview loading into druid jobs
......................................................................
Update oozie pageview loading into druid jobs
Make workflows easier to maintain using a variable instead
of repeating some complex value accross the files.
Bug: T156668
Change-Id: I401ed222b99036fdfca837c00ad558fa16b5854f
---
M oozie/pageview/druid/daily/workflow.xml
M oozie/pageview/druid/monthly/workflow.xml
2 files changed, 16 insertions(+), 8 deletions(-)
Approvals:
Mforns: Looks good to me, but someone else must approve
Nuria: Verified; Looks good to me, approved
diff --git a/oozie/pageview/druid/daily/workflow.xml
b/oozie/pageview/druid/daily/workflow.xml
index 3cf3296..39195d9 100644
--- a/oozie/pageview/druid/daily/workflow.xml
+++ b/oozie/pageview/druid/daily/workflow.xml
@@ -59,6 +59,10 @@
<description>A directory in HDFS for temporary files</description>
</property>
<property>
+ <name>temporary_dataset_path</name>
+
<value>${temporary_directory}/${wf:id()}-daily-druid-pageviews-${year}-${month}-${day}</value>
+ </property>
+ <property>
<name>load_druid_workflow_file</name>
<description>Workflow for loading druid</description>
</property>
@@ -107,7 +111,7 @@
</configuration>
<script>generate_daily_druid_pageviews.hql</script>
<param>source_table=${pageview_table}</param>
-
<param>destination_directory=${temporary_directory}/${wf:id()}-daily-druid-pageviews-${year}-${month}-${day}</param>
+ <param>destination_directory=${temporary_dataset_path}</param>
<param>year=${year}</param>
<param>month=${month}</param>
<param>day=${day}</param>
@@ -122,7 +126,7 @@
<configuration>
<property>
<name>directory</name>
-
<value>${temporary_directory}/${wf:id()}-daily-druid-pageviews-${year}-${month}-${day}</value>
+ <value>${temporary_dataset_path}</value>
</property>
</configuration>
</sub-workflow>
@@ -138,7 +142,7 @@
<configuration>
<property>
<name>source_directory</name>
-
<value>${temporary_directory}/${wf:id()}-daily-druid-pageviews-${year}-${month}-${day}</value>
+ <value>${temporary_dataset_path}</value>
</property>
<property>
<name>template_file</name>
@@ -160,7 +164,7 @@
<action name="remove_temporary_data">
<fs>
- <delete
path="${temporary_directory}/${wf:id()}-daily-druid-pageviews-${year}-${month}-${day}"/>
+ <delete path="${temporary_dataset_path}"/>
</fs>
<ok to="end"/>
<error to="send_error_email"/>
diff --git a/oozie/pageview/druid/monthly/workflow.xml
b/oozie/pageview/druid/monthly/workflow.xml
index 014d885..82ebd98 100644
--- a/oozie/pageview/druid/monthly/workflow.xml
+++ b/oozie/pageview/druid/monthly/workflow.xml
@@ -55,6 +55,10 @@
<description>A directory in HDFS for temporary files</description>
</property>
<property>
+ <name>temporary_dataset_path</name>
+
<value>${temporary_directory}/${wf:id()}-monthly-druid-pageviews-${year}-${month}</value>
+ </property>
+ <property>
<name>load_druid_workflow_file</name>
<description>Workflow for loading druid</description>
</property>
@@ -103,7 +107,7 @@
</configuration>
<script>generate_monthly_druid_pageviews.hql</script>
<param>source_table=${pageview_table}</param>
-
<param>destination_directory=${temporary_directory}/${wf:id()}-monthly-druid-pageviews-${year}-${month}</param>
+ <param>destination_directory=${temporary_dataset_path}</param>
<param>year=${year}</param>
<param>month=${month}</param>
</hive>
@@ -117,7 +121,7 @@
<configuration>
<property>
<name>directory</name>
-
<value>${temporary_directory}/${wf:id()}-monthly-druid-pageviews-${year}-${month}</value>
+ <value>${temporary_dataset_path}</value>
</property>
</configuration>
</sub-workflow>
@@ -133,7 +137,7 @@
<configuration>
<property>
<name>source_directory</name>
-
<value>${temporary_directory}/${wf:id()}-monthly-druid-pageviews-${year}-${month}</value>
+ <value>${temporary_dataset_path}</value>
</property>
<property>
<name>template_file</name>
@@ -155,7 +159,7 @@
<action name="remove_temporary_data">
<fs>
- <delete
path="${temporary_directory}/${wf:id()}-monthly-druid-pageviews-${year}-${month}"/>
+ <delete path="${temporary_dataset_path}"/>
</fs>
<ok to="end"/>
<error to="send_error_email"/>
--
To view, visit https://gerrit.wikimedia.org/r/335067
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I401ed222b99036fdfca837c00ad558fa16b5854f
Gerrit-PatchSet: 2
Gerrit-Project: analytics/refinery
Gerrit-Branch: master
Gerrit-Owner: Joal <[email protected]>
Gerrit-Reviewer: Joal <[email protected]>
Gerrit-Reviewer: Mforns <[email protected]>
Gerrit-Reviewer: Nuria <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
