Author: omalley
Date: Tue Mar 8 05:52:59 2011
New Revision: 1079183
URL: http://svn.apache.org/viewvc?rev=1079183&view=rev
Log:
commit db221627d41f6cb801bbf25da25fa6d822937d28
Author: Mahadev Konar <[email protected]>
Date: Fri Nov 12 06:29:25 2010 +0000
How to know what input was sent to a mapper? (log the input split where
users can see) from Krishna Ramachandran via mahadev)
+++ b/YAHOO-CHANGES.txt
+ How to know what input was sent to a mapper? (log the input
+ split where users can see) from
+
+ (Krishna Ramachandran via mahadev)
+
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java?rev=1079183&r1=1079182&r2=1079183&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java
Tue Mar 8 05:52:59 2011
@@ -58,6 +58,7 @@ import org.apache.hadoop.mapred.Merger.S
import org.apache.hadoop.mapred.SortedRanges.SkipRangeIterator;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.map.WrappedMapper;
+import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
import org.apache.hadoop.mapreduce.split.JobSplit;
import org.apache.hadoop.mapreduce.split.JobSplit.SplitMetaInfo;
import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitIndex;
@@ -417,6 +418,7 @@ class MapTask extends Task {
job.set(JobContext.MAP_INPUT_FILE, fileSplit.getPath().toString());
job.setLong(JobContext.MAP_INPUT_START, fileSplit.getStart());
job.setLong(JobContext.MAP_INPUT_PATH, fileSplit.getLength());
+ job.set(TTConfig.TT_MAP_INPUT_SPLITINFO, fileSplit.toString());
}
}
@@ -621,6 +623,7 @@ class MapTask extends Task {
split = getSplitDetails(new Path(splitIndex.getSplitLocation()),
splitIndex.getStartOffset());
+ job.set(TTConfig.TT_MAP_INPUT_SPLITINFO, split.toString());
org.apache.hadoop.mapreduce.RecordReader<INKEY,INVALUE> input =
new NewTrackingRecordReader<INKEY,INVALUE>
(inputFormat.createRecordReader(split, taskContext), reporter);
@@ -818,6 +821,7 @@ class MapTask extends Task {
LOG.info("soft limit at " + softLimit);
LOG.info("bufstart = " + bufstart + "; bufvoid = " + bufvoid);
LOG.info("kvstart = " + kvstart + "; length = " + maxRec);
+ LOG.info("split details: " + job.get(TTConfig.TT_MAP_INPUT_SPLITINFO));
}
// k/v serialization
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java?rev=1079183&r1=1079182&r2=1079183&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
Tue Mar 8 05:52:59 2011
@@ -70,6 +70,8 @@ public interface TTConfig extends MRConf
"mapreduce.tasktracker.instrumentation";
public static final String TT_MAP_SLOTS =
"mapreduce.tasktracker.map.tasks.maximum";
+ public static final String TT_MAP_INPUT_SPLITINFO =
+ "mapreduce.tasktracker.map.input.splitinfo";
/**
* @deprecated Use {@link #TT_RESOURCE_CALCULATOR_PLUGIN} instead
*/
