Author: cutting Date: Mon Jun 19 11:31:59 2006 New Revision: 415383 URL: http://svn.apache.org/viewvc?rev=415383&view=rev Log: HADOOP-299. Fix the tasktracker, permitting multiple jobs to more easily execute at the same time. Contributed by Owen.
Modified: lucene/hadoop/trunk/CHANGES.txt lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Modified: lucene/hadoop/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=415383&r1=415382&r2=415383&view=diff ============================================================================== --- lucene/hadoop/trunk/CHANGES.txt (original) +++ lucene/hadoop/trunk/CHANGES.txt Mon Jun 19 11:31:59 2006 @@ -6,6 +6,10 @@ 1. HADOOP-298. Improved progress reports for CopyFiles utility, the distributed file copier. (omalley via cutting) + 2. HADOOP-299. Fix some problems in the TaskTracker, permitting + multiple jobs to more easily execute at the same time. + (omalley via cutting) + Release 0.3.2 - 2006-06-09 Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=415383&r1=415382&r2=415383&view=diff ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Mon Jun 19 11:31:59 2006 @@ -39,7 +39,7 @@ static long RETIRE_JOB_CHECK_INTERVAL; static float TASK_ALLOC_EPSILON; static float PAD_FRACTION; - static float MIN_SLOTS_FOR_PADDING; + static final int MIN_CLUSTER_SIZE_FOR_PADDING = 3; /** * Used for formatting the id numbers @@ -405,8 +405,8 @@ RETIRE_JOB_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.interval", 24 * 60 * 60 * 1000); RETIRE_JOB_CHECK_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.check", 60 * 1000); TASK_ALLOC_EPSILON = conf.getFloat("mapred.jobtracker.taskalloc.loadbalance.epsilon", 0.2f); - PAD_FRACTION = conf.getFloat("mapred.jobtracker.taskalloc.capacitypad", 0.1f); - MIN_SLOTS_FOR_PADDING = 3 * maxCurrentTasks; + PAD_FRACTION = conf.getFloat("mapred.jobtracker.taskalloc.capacitypad", + 0.01f); // This is a directory of temporary submission files. We delete it // on startup, and can delete any files that we're done with @@ -643,8 +643,6 @@ // // Compute average map and reduce task numbers across pool // - int avgMaps = 0; - int avgReduces = 0; int remainingReduceLoad = 0; int remainingMapLoad = 0; int numTaskTrackers; @@ -669,8 +667,6 @@ } if (numTaskTrackers > 0) { - avgMaps = totalMaps / numTaskTrackers; - avgReduces = totalReduces / numTaskTrackers; avgMapLoad = remainingMapLoad / numTaskTrackers; avgReduceLoad = remainingReduceLoad / numTaskTrackers; } @@ -727,11 +723,12 @@ // schedule tasks to the hilt. // totalNeededMaps += job.desiredMaps(); - double padding = 0; - if (totalCapacity > MIN_SLOTS_FOR_PADDING) { - padding = Math.min(maxCurrentTasks, totalNeededMaps * PAD_FRACTION); + int padding = 0; + if (numTaskTrackers > MIN_CLUSTER_SIZE_FOR_PADDING) { + padding = Math.min(maxCurrentTasks, + (int)(totalNeededMaps * PAD_FRACTION)); } - if (totalNeededMaps + padding >= totalCapacity) { + if (totalMaps + padding >= totalCapacity) { break; } } @@ -762,11 +759,13 @@ // schedule tasks to the hilt. // totalNeededReduces += job.desiredReduces(); - double padding = 0; - if (totalCapacity > MIN_SLOTS_FOR_PADDING) { - padding = Math.min(maxCurrentTasks, totalNeededReduces * PAD_FRACTION); + int padding = 0; + if (numTaskTrackers > MIN_CLUSTER_SIZE_FOR_PADDING) { + padding = + Math.min(maxCurrentTasks, + (int) (totalNeededReduces * PAD_FRACTION)); } - if (totalNeededReduces + padding >= totalCapacity) { + if (totalReduces + padding >= totalCapacity) { break; } }