Author: tomwhite
Date: Thu Mar 15 00:32:15 2012
New Revision: 1300798
URL: http://svn.apache.org/viewvc?rev=1300798&view=rev
Log:
MAPREDUCE-2835. Make per-job counter limits configurable.
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/mapred/mapred-default.xml
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/Counters.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestUserDefinedCounters.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1300798&r1=1300797&r2=1300798&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Thu Mar 15 00:32:15 2012
@@ -157,6 +157,8 @@ Release 1.1.0 - unreleased
HDFS-3075. Backport HADOOP-4885: Try to restore failed name-node storage
directories at checkpoint time. (Brandon Li via szetszwo)
+ MAPREDUCE-2835. Make per-job counter limits configurable. (tomwhite)
+
Release 1.0.2 - unreleased
NEW FEATURES
Modified: hadoop/common/branches/branch-1/src/mapred/mapred-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/mapred-default.xml?rev=1300798&r1=1300797&r2=1300798&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/mapred-default.xml (original)
+++ hadoop/common/branches/branch-1/src/mapred/mapred-default.xml Thu Mar 15
00:32:15 2012
@@ -1233,13 +1233,36 @@
</description>
</property>
+<!-- end of node health script variables -->
+
<property>
- <name>mapreduce.job.counters.limit</name>
+ <name>mapreduce.job.counters.max</name>
<value>120</value>
<description>Limit on the number of counters allowed per job.
</description>
</property>
-<!-- end of node health script variables -->
+<property>
+ <name>mapreduce.job.counters.groups.max</name>
+ <value>50</value>
+ <description>Limit on the number of counter groups allowed per job.
+ </description>
+</property>
+
+<property>
+ <name>mapreduce.job.counters.counter.name.max</name>
+ <value>64</value>
+ <description>Limit on the length of counter names in jobs. Names
+ exceeding this limit will be truncated.
+ </description>
+</property>
+
+<property>
+ <name>mapreduce.job.counters.group.name.max</name>
+ <value>128</value>
+ <description>Limit on the length of counter group names in jobs. Names
+ exceeding this limit will be truncated.
+ </description>
+</property>
</configuration>
Modified:
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/Counters.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/Counters.java?rev=1300798&r1=1300797&r2=1300798&view=diff
==============================================================================
---
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/Counters.java
(original)
+++
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/Counters.java
Thu Mar 15 00:32:15 2012
@@ -59,18 +59,22 @@ public class Counters implements Writabl
private static char[] charsToEscape = {GROUP_OPEN, GROUP_CLOSE,
COUNTER_OPEN, COUNTER_CLOSE,
UNIT_OPEN, UNIT_CLOSE};
+ private static final JobConf conf = new JobConf();
/** limit on the size of the name of the group **/
- private static final int GROUP_NAME_LIMIT = 128;
+ private static final int GROUP_NAME_LIMIT =
+ conf.getInt("mapreduce.job.counters.group.name.max", 128);
/** limit on the size of the counter name **/
- private static final int COUNTER_NAME_LIMIT = 64;
+ private static final int COUNTER_NAME_LIMIT =
+ conf.getInt("mapreduce.job.counters.counter.name.max", 64);
- private static final JobConf conf = new JobConf();
/** limit on counters **/
public static int MAX_COUNTER_LIMIT =
- conf.getInt("mapreduce.job.counters.limit", 120);
+ conf.getInt("mapreduce.job.counters.limit", // deprecated in 0.23
+ conf.getInt("mapreduce.job.counters.max", 120));
/** the max groups allowed **/
- static final int MAX_GROUP_LIMIT = 50;
+ public static final int MAX_GROUP_LIMIT =
+ conf.getInt("mapreduce.job.counters.groups.max", 50);
/** the number of current counters**/
private int numCounters = 0;
Modified:
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestUserDefinedCounters.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestUserDefinedCounters.java?rev=1300798&r1=1300797&r2=1300798&view=diff
==============================================================================
---
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestUserDefinedCounters.java
(original)
+++
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestUserDefinedCounters.java
Thu Mar 15 00:32:15 2012
@@ -130,7 +130,7 @@ public class TestUserDefinedCounters ext
assertEquals(4,
runningJob.getCounters().getGroup("StringCounter")
.getCounter("MapRecords"));
- assertTrue(counters.getGroupNames().size() <= 51);
+ assertTrue(counters.getGroupNames().size() <= Counters.MAX_GROUP_LIMIT);
int i = 0;
while (counters.size() < Counters.MAX_COUNTER_LIMIT) {
counters.incrCounter("IncrCounter", "limit " + i, 2);
