Ottomata has submitted this change and it was merged.
Change subject: Refine upload webrequests
......................................................................
Refine upload webrequests
Change-Id: Ie67ebe264c3bd75df248c72b11d8ee9b8c89793b
---
M diagrams/oozie-overview.dia
M oozie/webrequest/datasets.xml
M oozie/webrequest/refine/bundle.xml
3 files changed, 9 insertions(+), 5 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git a/diagrams/oozie-overview.dia b/diagrams/oozie-overview.dia
index efdf32b..2944843 100644
--- a/diagrams/oozie-overview.dia
+++ b/diagrams/oozie-overview.dia
Binary files differ
diff --git a/oozie/webrequest/datasets.xml b/oozie/webrequest/datasets.xml
index ba2bc70..902a58c 100644
--- a/oozie/webrequest/datasets.xml
+++ b/oozie/webrequest/datasets.xml
@@ -17,7 +17,7 @@
storage format, and contain extra information. These
are sometimes also referred to as the 'refined webrequest'.
- This dataset does not yet include upload or bits.
+ This dataset does not yet include bits.
To unpad MONTH, DAY, and HOUR, we force coercion to a number by
adding 0.
@@ -54,5 +54,11 @@
<done-flag>_SUCCESS</done-flag>
</dataset>
-
+ <dataset name="webrequest_upload"
+ frequency="${coord:hours(1)}"
+ initial-instance="${start_time}"
+ timezone="Universal">
+
<uri-template>${webrequest_data_directory}/webrequest_source=upload/year=${YEAR}/month=${"$"}{MONTH
+ 0}/day=${"$"}{DAY + 0}/hour=${"$"}{HOUR + 0}</uri-template>
+ <done-flag>_SUCCESS</done-flag>
+ </dataset>
</datasets>
diff --git a/oozie/webrequest/refine/bundle.xml
b/oozie/webrequest/refine/bundle.xml
index 859c3d0..48ba5db 100644
--- a/oozie/webrequest/refine/bundle.xml
+++ b/oozie/webrequest/refine/bundle.xml
@@ -30,7 +30,7 @@
</parameters>
<!--
- For now, don't refine bits or upload.
+ For now, don't refine bits.
<coordinator name='refine-webrequest_bits'>
<app-path>${coordinator_file}</app-path>
@@ -73,7 +73,6 @@
</configuration>
</coordinator>
- <!--
<coordinator name='refine-webrequest_upload'>
<app-path>${coordinator_file}</app-path>
<configuration>
@@ -83,5 +82,4 @@
</property>
</configuration>
</coordinator>
- -->
</bundle-app>
--
To view, visit https://gerrit.wikimedia.org/r/186774
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ie67ebe264c3bd75df248c72b11d8ee9b8c89793b
Gerrit-PatchSet: 3
Gerrit-Project: analytics/refinery
Gerrit-Branch: master
Gerrit-Owner: QChris <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
