Joal has uploaded a new change for review.
https://gerrit.wikimedia.org/r/274157
Change subject: Correct mobile_apps uniques jobs
......................................................................
Correct mobile_apps uniques jobs
Remove hard-set number of reducers, single file is ensured
by ordering.
Set oozie parameter to let hive define the number of reducers
for middle steps.
Change-Id: I818c443c723a4f5aadd96e777397aa4c92562d4e
---
M oozie/mobile_apps/uniques/daily/generate_uniques_daily.hql
M oozie/mobile_apps/uniques/daily/workflow.xml
M oozie/mobile_apps/uniques/monthly/generate_uniques_monthly.hql
M oozie/mobile_apps/uniques/monthly/workflow.xml
4 files changed, 10 insertions(+), 7 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/refinery
refs/changes/57/274157/1
diff --git a/oozie/mobile_apps/uniques/daily/generate_uniques_daily.hql
b/oozie/mobile_apps/uniques/daily/generate_uniques_daily.hql
index b772171..19156d2 100644
--- a/oozie/mobile_apps/uniques/daily/generate_uniques_daily.hql
+++ b/oozie/mobile_apps/uniques/daily/generate_uniques_daily.hql
@@ -33,9 +33,6 @@
SET hive.exec.compress.output=true;
SET
mapreduce.output.fileoutputformat.compress.codec=org.apache.hadoop.io.compress.GzipCodec;
--- Hard-limiting number of reducer to force a single file in the target
directory.
-SET mapred.reduce.tasks=1;
-
-- Create a temporary table, then compute the new unique count
-- and concatenate it to archived data.
diff --git a/oozie/mobile_apps/uniques/daily/workflow.xml
b/oozie/mobile_apps/uniques/daily/workflow.xml
index d35ab15..b2d49a2 100644
--- a/oozie/mobile_apps/uniques/daily/workflow.xml
+++ b/oozie/mobile_apps/uniques/daily/workflow.xml
@@ -81,6 +81,11 @@
<name>mapreduce.job.queuename</name>
<value>${queue_name}</value>
</property>
+ <!--Let hive decide on the number of reducers -->
+ <property>
+ <name>mapred.reduce.tasks</name>
+ <value>-1</value>
+ </property>
<property>
<name>hive.exec.scratchdir</name>
<value>/tmp/hive-${user}</value>
diff --git a/oozie/mobile_apps/uniques/monthly/generate_uniques_monthly.hql
b/oozie/mobile_apps/uniques/monthly/generate_uniques_monthly.hql
index 1d263c0..4952049 100644
--- a/oozie/mobile_apps/uniques/monthly/generate_uniques_monthly.hql
+++ b/oozie/mobile_apps/uniques/monthly/generate_uniques_monthly.hql
@@ -31,10 +31,6 @@
SET hive.exec.compress.output=true;
SET
mapreduce.output.fileoutputformat.compress.codec=org.apache.hadoop.io.compress.GzipCodec;
--- Hard-limiting number of reducer to force a single file in the
--- target directory.
-SET mapred.reduce.tasks=1;
-
-- Create a temporary table, then compute the new unique count
-- and concatenate it to archived data.
diff --git a/oozie/mobile_apps/uniques/monthly/workflow.xml
b/oozie/mobile_apps/uniques/monthly/workflow.xml
index 01ef96d..f32631a 100644
--- a/oozie/mobile_apps/uniques/monthly/workflow.xml
+++ b/oozie/mobile_apps/uniques/monthly/workflow.xml
@@ -76,6 +76,11 @@
<name>mapreduce.job.queuename</name>
<value>${queue_name}</value>
</property>
+ <!--Let hive decide on the number of reducers -->
+ <property>
+ <name>mapred.reduce.tasks</name>
+ <value>-1</value>
+ </property>
<property>
<name>hive.exec.scratchdir</name>
<value>/tmp/hive-${user}</value>
--
To view, visit https://gerrit.wikimedia.org/r/274157
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I818c443c723a4f5aadd96e777397aa4c92562d4e
Gerrit-PatchSet: 1
Gerrit-Project: analytics/refinery
Gerrit-Branch: master
Gerrit-Owner: Joal <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
