Author: sharad
Date: Mon May 25 11:22:16 2009
New Revision: 778388
URL: http://svn.apache.org/viewvc?rev=778388&view=rev
Log:
HADOOP-5710. Counter MAP_INPUT_BYTES missing from new mapreduce api.
Contributed by Amareshwari Sriramadasu.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatistics.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Counters.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.properties
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/SequenceFileRecordReader.java
hadoop/core/trunk/src/test/findbugsExcludeFile.xml
hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCounters.java
hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon May 25 11:22:16 2009
@@ -673,6 +673,9 @@
HADOOP-5252. Streaming overrides -inputformat option (Klaas Bosteels
via sharad)
+ HADOOP-5710. Counter MAP_INPUT_BYTES missing from new mapreduce api.
+ (Amareshwari Sriramadasu via sharad)
+
Release 0.20.1 - Unreleased
INCOMPATIBLE CHANGES
Modified:
hadoop/core/trunk/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatistics.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatistics.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
---
hadoop/core/trunk/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatistics.java
(original)
+++
hadoop/core/trunk/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatistics.java
Mon May 25 11:22:16 2009
@@ -417,8 +417,6 @@
mapTask.setValue(MapTaskKeys.INPUT_RECORDS, parts[1]);
} else if (parts[0].equals("Map-Reduce Framework.Map output records"))
{
mapTask.setValue(MapTaskKeys.OUTPUT_RECORDS, parts[1]);
- } else if (parts[0].equals("Map-Reduce Framework.Map input bytes")) {
- mapTask.setValue(MapTaskKeys.INPUT_BYTES, parts[1]);
} else if (parts[0].equals("Map-Reduce Framework.Map output bytes")) {
mapTask.setValue(MapTaskKeys.OUTPUT_BYTES, parts[1]);
} else if (parts[0].equals("Map-Reduce Framework.Combine input
records")) {
@@ -427,6 +425,8 @@
mapTask.setValue(MapTaskKeys.COMBINE_OUTPUT_RECORDS, parts[1]);
} else if (parts[0].equals("Map-Reduce Framework.Spilled Records")) {
mapTask.setValue(MapTaskKeys.SPILLED_RECORDS, parts[1]);
+ } else if (parts[0].equals("FileInputFormatCounters.BYTES_READ")) {
+ mapTask.setValue(MapTaskKeys.INPUT_BYTES, parts[1]);
} else {
System.out.println("MapCounterKey:<"+parts[0]+"> ==> NOT INCLUDED IN
PERFORMANCE ADVISOR MAP TASK");
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Counters.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Counters.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Counters.java
(original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Counters.java Mon May
25 11:22:16 2009
@@ -36,6 +36,7 @@
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.StringUtils;
/**
@@ -398,6 +399,13 @@
* @return the counter for that name
*/
public synchronized Counter findCounter(String group, String name) {
+ if (name.equals("MAP_INPUT_BYTES")) {
+ group = FileInputFormat.COUNTER_GROUP;
+ name = FileInputFormat.BYTES_READ;
+ LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " +
+ "Use FileInputFormatCounters as group name and " +
+ " BYTES_READ as counter name instead");
+ }
return getGroup(group).getCounterForName(name);
}
@@ -411,7 +419,7 @@
*/
@Deprecated
public synchronized Counter findCounter(String group, int id, String name) {
- return getGroup(group).getCounterForName(name);
+ return findCounter(group, name);
}
/**
@@ -432,7 +440,7 @@
* @param amount amount by which counter is to be incremented
*/
public synchronized void incrCounter(String group, String counter, long
amount) {
- getGroup(group).getCounterForName(counter).increment(amount);
+ findCounter(group, counter).increment(amount);
}
/**
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java
(original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java Mon May
25 11:22:16 2009
@@ -51,6 +51,7 @@
import org.apache.hadoop.mapred.IFile.Writer;
import org.apache.hadoop.mapred.Merger.Segment;
import org.apache.hadoop.mapred.SortedRanges.SkipRangeIterator;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskCounter;
import org.apache.hadoop.util.IndexedSortable;
@@ -157,7 +158,9 @@
throws IOException{
rawIn = raw;
inputRecordCounter = reporter.getCounter(TaskCounter.MAP_INPUT_RECORDS);
- inputByteCounter = reporter.getCounter(TaskCounter.MAP_INPUT_BYTES);
+ inputByteCounter = reporter.getCounter(
+ FileInputFormat.COUNTER_GROUP,
+ FileInputFormat.BYTES_READ);
this.reporter = reporter;
}
Modified:
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.java
(original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.java
Mon May 25 11:22:16 2009
@@ -23,7 +23,6 @@
MAP_INPUT_RECORDS,
MAP_OUTPUT_RECORDS,
MAP_SKIPPED_RECORDS,
- MAP_INPUT_BYTES,
MAP_OUTPUT_BYTES,
COMBINE_INPUT_RECORDS,
COMBINE_OUTPUT_RECORDS,
Modified:
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.properties
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.properties?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
---
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.properties
(original)
+++
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.properties
Mon May 25 11:22:16 2009
@@ -3,7 +3,6 @@
CounterGroupName= Map-Reduce Framework
MAP_INPUT_RECORDS.name= Map input records
-MAP_INPUT_BYTES.name= Map input bytes
MAP_OUTPUT_RECORDS.name= Map output records
MAP_OUTPUT_BYTES.name= Map output bytes
MAP_SKIPPED_RECORDS.name= Map skipped records
Modified:
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
---
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
(original)
+++
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
Mon May 25 11:22:16 2009
@@ -49,6 +49,9 @@
* not split-up and are processed as a whole by {...@link Mapper}s.
*/
public abstract class FileInputFormat<K, V> extends InputFormat<K, V> {
+ public static final String COUNTER_GROUP =
+ "FileInputFormatCounters";
+ public static final String BYTES_READ = "BYTES_READ";
private static final Log LOG = LogFactory.getLog(FileInputFormat.class);
Modified:
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
---
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java
(original)
+++
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java
Mon May 25 11:22:16 2009
@@ -28,9 +28,11 @@
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.MapContext;
import org.apache.hadoop.util.LineReader;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.Log;
@@ -49,10 +51,13 @@
private int maxLineLength;
private LongWritable key = null;
private Text value = null;
+ private Counter inputByteCounter;
public void initialize(InputSplit genericSplit,
TaskAttemptContext context) throws IOException {
FileSplit split = (FileSplit) genericSplit;
+ inputByteCounter = ((MapContext)context).getCounter(
+ FileInputFormat.COUNTER_GROUP, FileInputFormat.BYTES_READ);
Configuration job = context.getConfiguration();
this.maxLineLength = job.getInt("mapred.linerecordreader.maxlength",
Integer.MAX_VALUE);
@@ -101,6 +106,7 @@
break;
}
pos += newSize;
+ inputByteCounter.increment(newSize);
if (newSize < maxLineLength) {
break;
}
Modified:
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/SequenceFileRecordReader.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/SequenceFileRecordReader.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
---
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/SequenceFileRecordReader.java
(original)
+++
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/SequenceFileRecordReader.java
Mon May 25 11:22:16 2009
@@ -25,7 +25,9 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.MapContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
@@ -38,12 +40,16 @@
private K key = null;
private V value = null;
protected Configuration conf;
+ private Counter inputByteCounter;
+ private long pos;
@Override
public void initialize(InputSplit split,
TaskAttemptContext context
) throws IOException, InterruptedException {
FileSplit fileSplit = (FileSplit) split;
+ inputByteCounter = ((MapContext)context).getCounter(
+ FileInputFormat.COUNTER_GROUP, FileInputFormat.BYTES_READ);
conf = context.getConfiguration();
Path path = fileSplit.getPath();
FileSystem fs = path.getFileSystem(conf);
@@ -64,7 +70,8 @@
if (!more) {
return false;
}
- long pos = in.getPosition();
+ inputByteCounter.increment(in.getPosition()-pos);
+ pos = in.getPosition();
key = (K) in.next(key);
if (key == null || (pos >= end && in.syncSeen())) {
more = false;
Modified: hadoop/core/trunk/src/test/findbugsExcludeFile.xml
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/findbugsExcludeFile.xml?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/findbugsExcludeFile.xml (original)
+++ hadoop/core/trunk/src/test/findbugsExcludeFile.xml Mon May 25 11:22:16 2009
@@ -84,6 +84,19 @@
<Bug pattern="BC_UNCONFIRMED_CAST" />
</Match>
<!--
+ We intentionally cast from a super class to sub class.
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.mapreduce.lib.input.LineRecordReader" />
+ <Method name="initialize" />
+ <Bug pattern="BC_UNCONFIRMED_CAST" />
+ </Match>
+ <Match>
+ <Class
name="org.apache.hadoop.mapreduce.lib.input.SequenceFileRecordReader" />
+ <Method name="initialize" />
+ <Bug pattern="BC_UNCONFIRMED_CAST" />
+ </Match>
+ <!--
We intentionally do the get name from the inner class
-->
<Match>
Modified:
hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCounters.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCounters.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
---
hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCounters.java
(original)
+++
hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCounters.java
Mon May 25 11:22:16 2009
@@ -70,7 +70,7 @@
}
public void testCounters() throws IOException {
- Enum[] keysWithResource = {TaskCounter.MAP_INPUT_BYTES,
+ Enum[] keysWithResource = {TaskCounter.MAP_INPUT_RECORDS,
TaskCounter.MAP_OUTPUT_BYTES};
Enum[] keysWithoutResource = {myCounters.TEST1, myCounters.TEST2};
Modified:
hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
---
hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
(original)
+++
hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
Mon May 25 11:22:16 2009
@@ -30,7 +30,6 @@
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.lib.NullOutputFormat;
-import org.apache.hadoop.mapreduce.TaskCounter;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.ToolRunner;
@@ -98,8 +97,10 @@
Sort sort = new Sort();
assertEquals(ToolRunner.run(job, sort, sortArgs), 0);
Counters counters = sort.getResult().getCounters();
- long mapInput = counters.findCounter(TaskCounter.MAP_INPUT_BYTES
- ).getValue();
+ long mapInput = counters.findCounter(
+ org.apache.hadoop.mapreduce.lib.input.FileInputFormat.COUNTER_GROUP,
+ org.apache.hadoop.mapreduce.lib.input.FileInputFormat.BYTES_READ).
+ getValue();
long hdfsRead = counters.findCounter(Task.FILESYSTEM_COUNTER_GROUP,
"HDFS_BYTES_READ").getValue();
// the hdfs read should be between 100% and 110% of the map input bytes
Modified:
hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
---
hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java
(original)
+++
hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java
Mon May 25 11:22:16 2009
@@ -118,6 +118,9 @@
out);
Counters ctrs = job.getCounters();
System.out.println("Counters: " + ctrs);
+ long mapIn = ctrs.findCounter(FileInputFormat.COUNTER_GROUP,
+ FileInputFormat.BYTES_READ).getValue();
+ assertTrue(mapIn != 0);
long combineIn = ctrs.findCounter(COUNTER_GROUP,
"COMBINE_INPUT_RECORDS").getValue();
long combineOut = ctrs.findCounter(COUNTER_GROUP,
