Author: tgraves Date: Wed Aug 29 15:38:40 2012 New Revision: 1378614 URL: http://svn.apache.org/viewvc?rev=1378614&view=rev Log: MAPREDUCE-4499. Looking for speculative tasks is very expensive in 1.x (Koji Noguchi via tgraves)
Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1378614&r1=1378613&r2=1378614&view=diff ============================================================================== --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Wed Aug 29 15:38:40 2012 @@ -103,6 +103,9 @@ Release 1.2.0 - unreleased MAPREDUCE-4408. allow jobs to set a JAR that is in the distributed cached (rkanter via tucu) + MAPREDUCE-4499. Looking for speculative tasks is very expensive in 1.x + (Koji Noguchi via tgraves) + OPTIMIZATIONS HDFS-2533. Backport: Remove needless synchronization on some FSDataSet Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1378614&r1=1378613&r2=1378614&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java (original) +++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java Wed Aug 29 15:38:40 2012 @@ -2246,21 +2246,19 @@ public class JobInProgress { continue; } - if (!tip.hasRunOnMachine(ttStatus.getHost(), + if (tip.hasSpeculativeTask(currentTime, avgProgress)) { + // Check if this tip can be removed from the list. + // If the list is shared then we should not remove. + if(shouldRemove){ + iter.remove(); + } + if (!tip.hasRunOnMachine(ttStatus.getHost(), ttStatus.getTrackerName())) { - if (tip.hasSpeculativeTask(currentTime, avgProgress)) { - // In case of shared list we don't remove it. Since the TIP failed - // on this tracker can be scheduled on some other tracker. - if (shouldRemove) { - iter.remove(); //this tracker is never going to run it again - } return tip; - } + } } else { - // Check if this tip can be removed from the list. - // If the list is shared then we should not remove. - if (shouldRemove) { - // This tracker will never speculate this tip + if (shouldRemove && tip.hasRunOnMachine(ttStatus.getHost(), + ttStatus.getTrackerName())) { iter.remove(); } }