Author: omalley
Date: Fri Mar 4 03:37:43 2011
New Revision: 1077073
URL: http://svn.apache.org/viewvc?rev=1077073&view=rev
Log:
commit 608ac20bc1d81d10a3e9cf01fd42eeb13d471d76
Author: Arun C Murthy <[email protected]>
Date: Fri Dec 11 13:31:45 2009 +0530
MAPREDUCE-1171. Allow the read-error notification in shuffle to be
configurable. Contributed by Amareshwari Sriramadasu.
From
https://issues.apache.org/jira/secure/attachment/12427571/patch-1171-1-ydist.txt
+++ b/YAHOO-CHANGES.txt
+ MAPREDUCE-1171. Allow the read-error notification in shuffle to be
+ configurable. (Amareshwari Sriramadasu via acmurthy)
+
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java?rev=1077073&r1=1077072&r2=1077073&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
Fri Mar 4 03:37:43 2011
@@ -895,6 +895,7 @@ class ReduceTask extends Task {
}
private int nextMapOutputCopierId = 0;
+ private boolean reportReadErrorImmediately;
/**
* Abstraction to track a map-output.
@@ -1836,6 +1837,8 @@ class ReduceTask extends Task {
);
this.random = new Random(randomSeed);
this.maxMapRuntime = 0;
+ this.reportReadErrorImmediately =
+ conf.getBoolean("mapreduce.reduce.shuffle.notify.readerror", true);
}
private boolean busyEnough(int numInFlight) {
@@ -2109,8 +2112,10 @@ class ReduceTask extends Task {
// using a hybrid technique for notifying the jobtracker.
// a. the first notification is sent after max-retries
// b. subsequent notifications are sent after 2 retries.
- // c. send notification immediately if it is a read error.
- if (cr.getError().equals(CopyOutputErrorType.READ_ERROR) ||
+ // c. send notification immediately if it is a read error and
+ // "mapreduce.reduce.shuffle.notify.readerror" set true.
+ if ((reportReadErrorImmediately && cr.getError().equals(
+ CopyOutputErrorType.READ_ERROR)) ||
((noFailedFetches >= fetchRetriesPerMap)
&& ((noFailedFetches - fetchRetriesPerMap) % 2) == 0)) {
synchronized (ReduceTask.this) {
