Ottomata has submitted this change and it was merged.
Change subject: Move Oozie's “marking directory done” into separate workflow
......................................................................
Move Oozie's “marking directory done” into separate workflow
With this refactor, the upcoming addition of webstatscollector can
reuse the code to mark its directory done upon success.
Change-Id: Ie557acff61b907e0a43c45f0ca82b5bf43a800d6
---
A oozie/util/mark_directory_done/workflow.xml
M oozie/webrequest/partition/add/bundle.properties
M oozie/webrequest/partition/add/coordinator.xml
M oozie/webrequest/partition/add/workflow.xml
4 files changed, 56 insertions(+), 3 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git a/oozie/util/mark_directory_done/workflow.xml
b/oozie/util/mark_directory_done/workflow.xml
new file mode 100644
index 0000000..e47050d
--- /dev/null
+++ b/oozie/util/mark_directory_done/workflow.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<workflow-app xmlns="uri:oozie:workflow:0.4"
+ name="mark-directory-done-wf-${directory}">
+
+ <parameters>
+ <property>
+ <name>done_file</name>
+ <value>_SUCCESS</value>
+ <description>
+ The name of the file to flag a directory as “done”.
+ </description>
+ </property>
+
+ <!-- Required properties -->
+ <property>
+ <name>directory</name>
+ <description>
+ The directory to mark “done”.
+
+ The done file will get generated in this directory.
+ </description>
+ </property>
+ </parameters>
+
+ <start to="mark_directory_done"/>
+
+ <action name="mark_directory_done">
+ <fs>
+ <touchz path="${directory}/${done_file}" />
+ </fs>
+ <ok to="end"/>
+ <error to="kill"/>
+ </action>
+
+ <kill name="kill">
+ <message>Action failed, error
message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+ </kill>
+ <end name="end"/>
+</workflow-app>
diff --git a/oozie/webrequest/partition/add/bundle.properties
b/oozie/webrequest/partition/add/bundle.properties
index 7a83eca..d7a8177 100644
--- a/oozie/webrequest/partition/add/bundle.properties
+++ b/oozie/webrequest/partition/add/bundle.properties
@@ -33,6 +33,9 @@
# Workflow to add a partition
add_partition_workflow_file =
${oozie_directory}/util/hive/partition/add/workflow.xml
+# Workflow to mark a directory as done
+mark_directory_done_workflow_file =
${oozie_directory}/util/mark_directory_done/workflow.xml
+
# HDFS path to hive-site.xml file. This is needed to run hive actions.
hive_site_xml = ${oozie_directory}/util/hive/hive-site.xml
diff --git a/oozie/webrequest/partition/add/coordinator.xml
b/oozie/webrequest/partition/add/coordinator.xml
index 14c9055..2f80ed0 100644
--- a/oozie/webrequest/partition/add/coordinator.xml
+++ b/oozie/webrequest/partition/add/coordinator.xml
@@ -26,6 +26,7 @@
<property><name>statistics_table</name></property>
<property><name>webrequest_source</name></property>
<property><name>faulty_hosts_directory</name></property>
+ <property><name>mark_directory_done_workflow_file</name></property>
</parameters>
<controls>
diff --git a/oozie/webrequest/partition/add/workflow.xml
b/oozie/webrequest/partition/add/workflow.xml
index b3287c2..10779c8 100644
--- a/oozie/webrequest/partition/add/workflow.xml
+++ b/oozie/webrequest/partition/add/workflow.xml
@@ -61,6 +61,10 @@
obviously faulty hosts will get collected.
</description>
</property>
+ <property>
+ <name>mark_directory_done_workflow_file</name>
+ <description>Workflow for marking a directory done</description>
+ </property>
</parameters>
<start to="add_partition"/>
@@ -128,9 +132,15 @@
</action>
<action name="mark_dataset_done">
- <fs>
- <touchz path="${location}/_SUCCESS" />
- </fs>
+ <sub-workflow>
+ <app-path>${mark_directory_done_workflow_file}</app-path>
+ <configuration>
+ <property>
+ <name>directory</name>
+ <value>${location}</value>
+ </property>
+ </configuration>
+ </sub-workflow>
<ok to="end"/>
<error to="kill"/>
</action>
--
To view, visit https://gerrit.wikimedia.org/r/161905
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ie557acff61b907e0a43c45f0ca82b5bf43a800d6
Gerrit-PatchSet: 2
Gerrit-Project: analytics/refinery
Gerrit-Branch: master
Gerrit-Owner: QChris <christ...@quelltextlich.at>
Gerrit-Reviewer: Ottomata <o...@wikimedia.org>
Gerrit-Reviewer: QChris <christ...@quelltextlich.at>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
