Joal has uploaded a new change for review.
https://gerrit.wikimedia.org/r/285388
Change subject: Add refinery_directory prop to every oozie job
......................................................................
Add refinery_directory prop to every oozie job
Change-Id: I8af83fd7f9f53a94862616714ad79503f7ce326f
---
M oozie/aqs/hourly/coordinator.properties
M oozie/mediacounts/archive/coordinator.properties
M oozie/mediawiki/load/bundle.properties
M oozie/mobile_apps/uniques/daily/coordinator.properties
M oozie/mobile_apps/uniques/monthly/coordinator.properties
M oozie/pagecounts-all-sites/archive/bundle.properties
M oozie/pagecounts-all-sites/load/coordinator.properties
M oozie/pageview/hourly/coordinator.properties
M oozie/util/hive/partition/add/workflow.properties
M oozie/webrequest/load/bundle.properties
10 files changed, 88 insertions(+), 10 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/refinery
refs/changes/88/285388/1
diff --git a/oozie/aqs/hourly/coordinator.properties
b/oozie/aqs/hourly/coordinator.properties
index d79870f..cee42c2 100644
--- a/oozie/aqs/hourly/coordinator.properties
+++ b/oozie/aqs/hourly/coordinator.properties
@@ -14,9 +14,16 @@
user = hdfs
+# Base path in HDFS to refinery.
+# When submitting this job for production, you should
+# override this to point directly at a deployed
+# directory name, and not the 'symbolic' 'current' directory.
+# E.g. /wmf/refinery/2015-01-05T17.59.18Z--7bb7f07
+refinery_directory = ${name_node}/wmf/refinery/current
+
# Base path in HDFS to oozie files.
# Other files will be used relative to this path.
-oozie_directory = ${name_node}/wmf/refinery/current/oozie
+oozie_directory = ${refinery_directory}/oozie
# HDFS path to coordinator to run.
coordinator_file =
${oozie_directory}/aqs/hourly/coordinator.xml
diff --git a/oozie/mediacounts/archive/coordinator.properties
b/oozie/mediacounts/archive/coordinator.properties
index db736cb..7e80123 100644
--- a/oozie/mediacounts/archive/coordinator.properties
+++ b/oozie/mediacounts/archive/coordinator.properties
@@ -12,9 +12,16 @@
job_tracker =
resourcemanager.analytics.eqiad.wmnet:8032
queue_name = default
+# Base path in HDFS to refinery.
+# When submitting this job for production, you should
+# override this to point directly at a deployed
+# directory name, and not the 'symbolic' 'current' directory.
+# E.g. /wmf/refinery/2015-01-05T17.59.18Z--7bb7f07
+refinery_directory = ${name_node}/wmf/refinery/current
+
# Base path in HDFS to oozie files.
# Other files will be used relative to this path.
-oozie_directory = ${name_node}/wmf/refinery/current/oozie
+oozie_directory = ${refinery_directory}/oozie
# HDFS path to workflow to run.
workflow_file =
${oozie_directory}/mediacounts/archive/workflow.xml
diff --git a/oozie/mediawiki/load/bundle.properties
b/oozie/mediawiki/load/bundle.properties
index c6c2605..8ca0138 100644
--- a/oozie/mediawiki/load/bundle.properties
+++ b/oozie/mediawiki/load/bundle.properties
@@ -13,9 +13,16 @@
queue_name = default
user = hdfs
+# Base path in HDFS to refinery.
+# When submitting this job for production, you should
+# override this to point directly at a deployed
+# directory name, and not the 'symbolic' 'current' directory.
+# E.g. /wmf/refinery/2015-01-05T17.59.18Z--7bb7f07
+refinery_directory = ${name_node}/wmf/refinery/current
+
# Base path in HDFS to oozie files.
# Other files will be used relative to this path.
-oozie_directory = ${name_node}/wmf/refinery/current/oozie
+oozie_directory = ${refinery_directory}/oozie
# HDFS path to coordinator to run for each webrequest_source.
coordinator_file =
${oozie_directory}/mediawiki/load/coordinator.xml
diff --git a/oozie/mobile_apps/uniques/daily/coordinator.properties
b/oozie/mobile_apps/uniques/daily/coordinator.properties
index bb50e97..0861cb5 100644
--- a/oozie/mobile_apps/uniques/daily/coordinator.properties
+++ b/oozie/mobile_apps/uniques/daily/coordinator.properties
@@ -2,7 +2,16 @@
job_tracker = resourcemanager.analytics.eqiad.wmnet:8032
queue_name = default
-oozie_directory = ${name_node}/wmf/refinery/current/oozie
+# Base path in HDFS to refinery.
+# When submitting this job for production, you should
+# override this to point directly at a deployed
+# directory name, and not the 'symbolic' 'current' directory.
+# E.g. /wmf/refinery/2015-01-05T17.59.18Z--7bb7f07
+refinery_directory = ${name_node}/wmf/refinery/current
+
+# Base path in HDFS to oozie files.
+# Other files will be used relative to this path.
+oozie_directory = ${refinery_directory}/oozie
hive_site_xml = ${oozie_directory}/util/hive/hive-site.xml
# Workflow app to run.
diff --git a/oozie/mobile_apps/uniques/monthly/coordinator.properties
b/oozie/mobile_apps/uniques/monthly/coordinator.properties
index 9923df0..58f9075 100644
--- a/oozie/mobile_apps/uniques/monthly/coordinator.properties
+++ b/oozie/mobile_apps/uniques/monthly/coordinator.properties
@@ -5,7 +5,16 @@
job_tracker = resourcemanager.analytics.eqiad.wmnet:8032
queue_name = default
-oozie_directory = ${name_node}/wmf/refinery/current/oozie
+# Base path in HDFS to refinery.
+# When submitting this job for production, you should
+# override this to point directly at a deployed
+# directory name, and not the 'symbolic' 'current' directory.
+# E.g. /wmf/refinery/2015-01-05T17.59.18Z--7bb7f07
+refinery_directory = ${name_node}/wmf/refinery/current
+
+# Base path in HDFS to oozie files.
+# Other files will be used relative to this path.
+oozie_directory = ${refinery_directory}/oozie
hive_site_xml = ${oozie_directory}/util/hive/hive-site.xml
# Workflow app to run.
diff --git a/oozie/pagecounts-all-sites/archive/bundle.properties
b/oozie/pagecounts-all-sites/archive/bundle.properties
index 4f874b6..e6dbb6c 100644
--- a/oozie/pagecounts-all-sites/archive/bundle.properties
+++ b/oozie/pagecounts-all-sites/archive/bundle.properties
@@ -13,9 +13,16 @@
job_tracker =
resourcemanager.analytics.eqiad.wmnet:8032
queue_name = default
+# Base path in HDFS to refinery.
+# When submitting this job for production, you should
+# override this to point directly at a deployed
+# directory name, and not the 'symbolic' 'current' directory.
+# E.g. /wmf/refinery/2015-01-05T17.59.18Z--7bb7f07
+refinery_directory = ${name_node}/wmf/refinery/current
+
# Base path in HDFS to oozie files.
# Other files will be used relative to this path.
-oozie_directory =
${name_node}/wmf/refinery/current/oozie
+oozie_directory = ${refinery_directory}/oozie
# HDFS path to coordinator to run for each webrequest_source.
coordinator_file =
${oozie_directory}/pagecounts-all-sites/archive/coordinator.xml
diff --git a/oozie/pagecounts-all-sites/load/coordinator.properties
b/oozie/pagecounts-all-sites/load/coordinator.properties
index dd3678e..6093ee8 100644
--- a/oozie/pagecounts-all-sites/load/coordinator.properties
+++ b/oozie/pagecounts-all-sites/load/coordinator.properties
@@ -13,9 +13,16 @@
job_tracker =
resourcemanager.analytics.eqiad.wmnet:8032
queue_name = default
+# Base path in HDFS to refinery.
+# When submitting this job for production, you should
+# override this to point directly at a deployed
+# directory name, and not the 'symbolic' 'current' directory.
+# E.g. /wmf/refinery/2015-01-05T17.59.18Z--7bb7f07
+refinery_directory = ${name_node}/wmf/refinery/current
+
# Base path in HDFS to oozie files.
# Other files will be used relative to this path.
-oozie_directory = ${name_node}/wmf/refinery/current/oozie
+oozie_directory = ${refinery_directory}/oozie
# HDFS path to workflow to run.
workflow_file =
${oozie_directory}/pagecounts-all-sites/load/workflow.xml
diff --git a/oozie/pageview/hourly/coordinator.properties
b/oozie/pageview/hourly/coordinator.properties
index f8c7eeb..4e96ca9 100644
--- a/oozie/pageview/hourly/coordinator.properties
+++ b/oozie/pageview/hourly/coordinator.properties
@@ -14,9 +14,16 @@
user = hdfs
+# Base path in HDFS to refinery.
+# When submitting this job for production, you should
+# override this to point directly at a deployed
+# directory name, and not the 'symbolic' 'current' directory.
+# E.g. /wmf/refinery/2015-01-05T17.59.18Z--7bb7f07
+refinery_directory = ${name_node}/wmf/refinery/current
+
# Base path in HDFS to oozie files.
# Other files will be used relative to this path.
-oozie_directory = ${name_node}/wmf/refinery/current/oozie
+oozie_directory = ${refinery_directory}/oozie
# HDFS path to coordinator to run for each webrequest_source.
coordinator_file =
${oozie_directory}/pageview/hourly/coordinator.xml
diff --git a/oozie/util/hive/partition/add/workflow.properties
b/oozie/util/hive/partition/add/workflow.properties
index 116da72..1925391 100644
--- a/oozie/util/hive/partition/add/workflow.properties
+++ b/oozie/util/hive/partition/add/workflow.properties
@@ -10,9 +10,16 @@
job_tracker = resourcemanager.analytics.eqiad.wmnet:8032
queue_name = default
+# Base path in HDFS to refinery.
+# When submitting this job for production, you should
+# override this to point directly at a deployed
+# directory name, and not the 'symbolic' 'current' directory.
+# E.g. /wmf/refinery/2015-01-05T17.59.18Z--7bb7f07
+refinery_directory = ${name_node}/wmf/refinery/current
+
# Base path in HDFS to oozie files.
# Other files will be used relative to this path.
-oozie_directory = ${name_node}/wmf/refinery/current/oozie
+oozie_directory = ${refinery_directory}/oozie
# HDFS path to hive-site.xml file. This is needed to run hive actions.
hive_site_xml = ${oozie_directory}/util/hive/hive-site.xml
diff --git a/oozie/webrequest/load/bundle.properties
b/oozie/webrequest/load/bundle.properties
index 65473c4..5846e1e 100644
--- a/oozie/webrequest/load/bundle.properties
+++ b/oozie/webrequest/load/bundle.properties
@@ -11,9 +11,20 @@
job_tracker = resourcemanager.analytics.eqiad.wmnet:8032
queue_name = default
+# Base path in HDFS to refinery.
+# When submitting this job for production, you should
+# override this to point directly at a deployed
+# directory name, and not the 'symbolic' 'current' directory.
+# E.g. /wmf/refinery/2015-01-05T17.59.18Z--7bb7f07
+refinery_directory = ${name_node}/wmf/refinery/current
+
+# HDFS path to artifacts that will be used by this job.
+# E.g. refinery-hive.jar should exist here.
+artifacts_directory = ${refinery_directory}/artifacts
+
# Base path in HDFS to oozie files.
# Other files will be used relative to this path.
-oozie_directory = ${name_node}/wmf/refinery/current/oozie
+oozie_directory = ${refinery_directory}/oozie
# HDFS path to coordinator to run for each webrequest_source.
coordinator_file =
${oozie_directory}/webrequest/load/coordinator.xml
--
To view, visit https://gerrit.wikimedia.org/r/285388
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I8af83fd7f9f53a94862616714ad79503f7ce326f
Gerrit-PatchSet: 1
Gerrit-Project: analytics/refinery
Gerrit-Branch: master
Gerrit-Owner: Joal <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
