Author: daijy
Date: Fri May 1 22:42:34 2015
New Revision: 1677247
URL: http://svn.apache.org/r1677247
Log:
PIG-4529: Pig on tez hit counter limit imposed by MR
Modified:
pig/branches/branch-0.15/CHANGES.txt
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
Modified: pig/branches/branch-0.15/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.15/CHANGES.txt?rev=1677247&r1=1677246&r2=1677247&view=diff
==============================================================================
--- pig/branches/branch-0.15/CHANGES.txt (original)
+++ pig/branches/branch-0.15/CHANGES.txt Fri May 1 22:42:34 2015
@@ -66,6 +66,8 @@ PIG-4333: Split BigData tests into multi
BUG FIXES
+PIG-4529: Pig on tez hit counter limit imposed by MR (daijy)
+
PIG-4524: Pig Minicluster unit tests broken by TEZ-2333 (daijy)
PIG-4527: NON-ASCII Characters in Javadoc break 'ant docs' (nielsbasjes via
daijy)
Modified:
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java?rev=1677247&r1=1677246&r2=1677247&view=diff
==============================================================================
---
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
(original)
+++
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
Fri May 1 22:42:34 2015
@@ -34,6 +34,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.counters.Limits;
import org.apache.pig.PigConfiguration;
import org.apache.pig.PigException;
import org.apache.pig.PigWarning;
@@ -116,6 +117,18 @@ public class TezLauncher extends Launche
pc.getProperties().setProperty(TezConfiguration.TEZ_AM_DAG_SCHEDULER_CLASS,
DAGSchedulerNaturalOrderControlled.class.getName());
}
Configuration conf =
ConfigurationUtil.toConfiguration(pc.getProperties(), true);
+ // Make sure MR counter does not exceed limit
+ if (conf.get(TezConfiguration.TEZ_COUNTERS_MAX) != null) {
+
conf.setInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTER_GROUPS_MAX_KEY,
Math.max(
+
conf.getInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTER_GROUPS_MAX_KEY, 0),
+ conf.getInt(TezConfiguration.TEZ_COUNTERS_MAX, 0)));
+ }
+ if (conf.get(TezConfiguration.TEZ_COUNTERS_MAX_GROUPS) != null) {
+
conf.setInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTERS_MAX_KEY, Math.max(
+
conf.getInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTERS_MAX_KEY, 0),
+ conf.getInt(TezConfiguration.TEZ_COUNTERS_MAX_GROUPS, 0)));
+ }
+ Limits.init(conf);
if (pc.defaultParallel == -1 &&
!conf.getBoolean(PigConfiguration.PIG_TEZ_AUTO_PARALLELISM, true)) {
pc.defaultParallel = 1;
}
