Author: cutting Date: Mon Jun 26 11:37:38 2006 New Revision: 417274 URL: http://svn.apache.org/viewvc?rev=417274&view=rev Log: HADOOP-305. Fix tasktracker to ask for new tasks as soon as a task is finished. Contributed by Mahadev.
Modified: lucene/hadoop/trunk/CHANGES.txt lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java Modified: lucene/hadoop/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=417274&r1=417273&r2=417274&view=diff ============================================================================== --- lucene/hadoop/trunk/CHANGES.txt (original) +++ lucene/hadoop/trunk/CHANGES.txt Mon Jun 26 11:37:38 2006 @@ -52,8 +52,13 @@ error. (omalley via cutting) 13. HADOOP-304. Improve error message for - UnregisterdDatanodeException, adding expected node name. + UnregisterdDatanodeException to include expected node name. (Konstantin Shvachko via cutting) + +14. HADOOP-305. Fix TaskTracker to ask for new tasks as soon as a + task is finished, rather than waiting for the next heartbeat. + This improves performance when tasks are short. + (Mahadev Konar via cutting) Release 0.3.2 - 2006-06-09 Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=417274&r1=417273&r2=417274&view=diff ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Mon Jun 26 11:37:38 2006 @@ -79,7 +79,7 @@ private int maxCurrentTasks; private int failures; - + private int finishedCount[] = new int[1]; /** * A list of tips that should be cleaned up. */ @@ -284,11 +284,16 @@ long waitTime = HEARTBEAT_INTERVAL - (now - lastHeartbeat); if (waitTime > 0) { try { - Thread.sleep(waitTime); + // sleeps for the wait time, wakes up if a task is finished. + synchronized(finishedCount) { + if (finishedCount[0] == 0) { + finishedCount.wait(waitTime); + } + finishedCount[0] = 0; + } } catch (InterruptedException ie) { - } - continue; - } + } + } lastHeartbeat = now; // @@ -856,6 +861,10 @@ } if (tip != null) { tip.taskFinished(); + synchronized(finishedCount) { + finishedCount[0]++; + finishedCount.notifyAll(); + } } else { LOG.warn("Unknown child task finshed: "+taskid+". Ignored."); } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java?rev=417274&r1=417273&r2=417274&view=diff ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java Mon Jun 26 11:37:38 2006 @@ -114,10 +114,12 @@ int mapCount = 0; for (Iterator it = taskReports.iterator(); it.hasNext(); ) { TaskStatus ts = (TaskStatus) it.next(); - if (ts.getIsMap()) { - mapCount++; - } - } + if (ts.getIsMap() && + ((ts.getRunState() == TaskStatus.RUNNING) + || (ts.getRunState() == TaskStatus.UNASSIGNED))) { + mapCount++; + } + } return mapCount; } @@ -125,7 +127,16 @@ * Return the current ReduceTask count */ public int countReduceTasks() { - return taskReports.size() - countMapTasks(); + int reduceCount = 0; + for (Iterator it = taskReports.iterator(); it.hasNext(); ) { + TaskStatus ts = (TaskStatus) it.next(); + if ((!ts.getIsMap()) && + ((ts.getRunState() == TaskStatus.RUNNING) + || (ts.getRunState() == TaskStatus.UNASSIGNED))) { + reduceCount++; + } + } + return reduceCount; } /**