Author: daijy
Date: Tue Dec 22 19:21:52 2015
New Revision: 1721449
URL: http://svn.apache.org/viewvc?rev=1721449&view=rev
Log:
PIG-4760: TezDAGStats.convertToHadoopCounters is not used, but impose MR
counter limit
Modified:
pig/branches/branch-0.15/CHANGES.txt
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
Modified: pig/branches/branch-0.15/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.15/CHANGES.txt?rev=1721449&r1=1721448&r2=1721449&view=diff
==============================================================================
--- pig/branches/branch-0.15/CHANGES.txt (original)
+++ pig/branches/branch-0.15/CHANGES.txt Tue Dec 22 19:21:52 2015
@@ -28,6 +28,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4760: TezDAGStats.convertToHadoopCounters is not used, but impose MR
counter limit (daijy)
+
PIG-4696: Empty map returned by a streaming_python udf wrongly contains a null
key (cheolsoo)
PIG-4712: [Pig on Tez] NPE in Bloom UDF after Union (rohini)
Modified:
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java?rev=1721449&r1=1721448&r2=1721449&view=diff
==============================================================================
---
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
(original)
+++
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
Tue Dec 22 19:21:52 2015
@@ -19,6 +19,7 @@ package org.apache.pig.backend.hadoop.ex
import java.io.IOException;
import java.io.PrintStream;
+import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -121,16 +122,26 @@ public class TezLauncher extends Launche
Configuration conf =
ConfigurationUtil.toConfiguration(pc.getProperties(), true);
// Make sure MR counter does not exceed limit
if (conf.get(TezConfiguration.TEZ_COUNTERS_MAX) != null) {
-
conf.setInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTER_GROUPS_MAX_KEY,
Math.max(
-
conf.getInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTER_GROUPS_MAX_KEY, 0),
+
conf.setInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTERS_MAX_KEY, Math.max(
+
conf.getInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTERS_MAX_KEY, 0),
conf.getInt(TezConfiguration.TEZ_COUNTERS_MAX, 0)));
}
if (conf.get(TezConfiguration.TEZ_COUNTERS_MAX_GROUPS) != null) {
-
conf.setInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTERS_MAX_KEY, Math.max(
-
conf.getInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTERS_MAX_KEY, 0),
+
conf.setInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTER_GROUPS_MAX_KEY,
Math.max(
+
conf.getInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTER_GROUPS_MAX_KEY, 0),
conf.getInt(TezConfiguration.TEZ_COUNTERS_MAX_GROUPS, 0)));
}
- Limits.init(conf);
+
+ // This is hacky, but Limits cannot be initialized twice
+ try {
+ Field f = Limits.class.getDeclaredField("isInited");
+ f.setAccessible(true);
+ f.setBoolean(null, false);
+ Limits.init(conf);
+ } catch (Throwable e) {
+ log.warn("Error when setting counter limit: " + e.getMessage());
+ }
+
if (pc.defaultParallel == -1 &&
!conf.getBoolean(PigConfiguration.PIG_TEZ_AUTO_PARALLELISM, true)) {
pc.defaultParallel = 1;
}
