Author: acmurthy
Date: Wed Nov 21 04:35:27 2007
New Revision: 597057
URL: http://svn.apache.org/viewvc?rev=597057&view=rev
Log:
HADOOP-1274. Support different number of mappers and reducers per TaskTracker
to allow administrators to better configure and utilize heterogenous clusters.
Configuration changes to hadoop-default.xml:
add mapred.tasktracker.map.tasks.maximum (default value of 2)
add mapred.tasktracker.reduce.tasks.maximum (default value of 2)
remove mapred.tasktracker.tasks.maximum (deprecated for 0.16.0)
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/conf/hadoop-default.xml
lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/Sort.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ClusterStatus.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobSubmissionProtocol.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/SortValidator.java
lucene/hadoop/trunk/src/webapps/job/jobtracker.jsp
lucene/hadoop/trunk/src/webapps/job/machines.jsp
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=597057&r1=597056&r2=597057&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed Nov 21 04:35:27 2007
@@ -51,6 +51,14 @@
HADOOP-1989. Support simulated DataNodes. This helps creating large virtual
clusters for testing purposes. (Sanjay Radia via dhruba)
+ HADOOP-1274. Support different number of mappers and reducers per
+ TaskTracker to allow administrators to better configure and utilize
+ heterogenous clusters.
+ Configuration changes to hadoop-default.xml:
+ add mapred.tasktracker.map.tasks.maximum (default value of 2)
+ add mapred.tasktracker.reduce.tasks.maximum (default value of 2)
+ remove mapred.tasktracker.tasks.maximum (deprecated for 0.16.0)
+
OPTIMIZATIONS
HADOOP-1898. Release the lock protecting the last time of the last stack
Modified: lucene/hadoop/trunk/conf/hadoop-default.xml
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/conf/hadoop-default.xml?rev=597057&r1=597056&r2=597057&view=diff
==============================================================================
--- lucene/hadoop/trunk/conf/hadoop-default.xml (original)
+++ lucene/hadoop/trunk/conf/hadoop-default.xml Wed Nov 21 04:35:27 2007
@@ -603,9 +603,17 @@
</property>
<property>
- <name>mapred.tasktracker.tasks.maximum</name>
+ <name>mapred.tasktracker.map.tasks.maximum</name>
<value>2</value>
- <description>The maximum number of tasks that will be run
+ <description>The maximum number of map tasks that will be run
+ simultaneously by a task tracker.
+ </description>
+</property>
+
+<property>
+ <name>mapred.tasktracker.reduce.tasks.maximum</name>
+ <value>2</value>
+ <description>The maximum number of reduce tasks that will be run
simultaneously by a task tracker.
</description>
</property>
Modified: lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/Sort.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/Sort.java?rev=597057&r1=597056&r2=597057&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/Sort.java
(original)
+++ lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/Sort.java Wed
Nov 21 04:35:27 2007
@@ -76,8 +76,12 @@
ClusterStatus cluster = client.getClusterStatus();
int num_maps = cluster.getTaskTrackers() *
jobConf.getInt("test.sort.maps_per_host", 10);
- int num_reduces = cluster.getTaskTrackers() *
- jobConf.getInt("test.sort.reduces_per_host", cluster.getMaxTasks());
+ int num_reduces = (int) (cluster.getMaxReduceTasks() * 0.9);
+ String sort_reduces = jobConf.get("test.sort.reduces_per_host");
+ if (sort_reduces != null) {
+ num_reduces = cluster.getTaskTrackers() *
+ Integer.parseInt(sort_reduces);
+ }
Class<? extends InputFormat> inputFormatClass =
SequenceFileInputFormat.class;
Class<? extends OutputFormat> outputFormatClass =
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ClusterStatus.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ClusterStatus.java?rev=597057&r1=597056&r2=597057&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ClusterStatus.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ClusterStatus.java
Wed Nov 21 04:35:27 2007
@@ -54,7 +54,8 @@
private int task_trackers;
private int map_tasks;
private int reduce_tasks;
- private int max_tasks;
+ private int max_map_tasks;
+ private int max_reduce_tasks;
private JobTracker.State state;
ClusterStatus() {}
@@ -68,12 +69,13 @@
* @param max the maximum no. of tasks in the cluster
* @param state the [EMAIL PROTECTED] JobTracker.State} of the
<code>JobTracker</code>
*/
- ClusterStatus(int trackers, int maps, int reduces, int max,
- JobTracker.State state) {
+ ClusterStatus(int trackers, int maps, int reduces, int maxMaps,
+ int maxReduces, JobTracker.State state) {
task_trackers = trackers;
map_tasks = maps;
reduce_tasks = reduces;
- max_tasks = max;
+ max_map_tasks = maxMaps;
+ max_reduce_tasks = maxReduces;
this.state = state;
}
@@ -109,12 +111,32 @@
* Get the maximum capacity for running tasks in the cluster.
*
* @return the maximum capacity for running tasks in the cluster.
+ * @deprecated Use [EMAIL PROTECTED] #getMaxMapTasks()} and/or
+ * [EMAIL PROTECTED] #getMaxReduceTasks()}
*/
public int getMaxTasks() {
- return max_tasks;
+ return (max_map_tasks + max_reduce_tasks);
+ }
+
+ /**
+ * Get the maximum capacity for running map tasks in the cluster.
+ *
+ * @return the maximum capacity for running map tasks in the cluster.
+ */
+ public int getMaxMapTasks() {
+ return max_map_tasks;
}
/**
+ * Get the maximum capacity for running reduce tasks in the cluster.
+ *
+ * @return the maximum capacity for running reduce tasks in the cluster.
+ */
+ public int getMaxReduceTasks() {
+ return max_reduce_tasks;
+ }
+
+ /**
* Get the current state of the <code>JobTracker</code>,
* as [EMAIL PROTECTED] JobTracker.State}
*
@@ -128,7 +150,8 @@
out.writeInt(task_trackers);
out.writeInt(map_tasks);
out.writeInt(reduce_tasks);
- out.writeInt(max_tasks);
+ out.writeInt(max_map_tasks);
+ out.writeInt(max_reduce_tasks);
WritableUtils.writeEnum(out, state);
}
@@ -136,7 +159,8 @@
task_trackers = in.readInt();
map_tasks = in.readInt();
reduce_tasks = in.readInt();
- max_tasks = in.readInt();
+ max_map_tasks = in.readInt();
+ max_reduce_tasks = in.readInt();
state = WritableUtils.readEnum(in, JobTracker.State.class);
}
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java?rev=597057&r1=597056&r2=597057&view=diff
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java
Wed Nov 21 04:35:27 2007
@@ -35,8 +35,10 @@
* version 5 introduced that removes locateMapOutputs and instead uses
* getTaskCompletionEvents to figure finished maps and fetch the outputs
* version 6 adds maxTasks to TaskTrackerStatus for HADOOP-1245
+ * version 7 replaces maxTasks by maxMapTasks and maxReduceTasks in
+ * TaskTrackerStatus for HADOOP-1274
*/
- public static final long versionID = 6L;
+ public static final long versionID = 7L;
public final static int TRACKERS_OK = 0;
public final static int UNKNOWN_TASKTRACKER = 1;
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?rev=597057&r1=597056&r2=597057&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Wed Nov
21 04:35:27 2007
@@ -931,8 +931,8 @@
*
* <p>The right number of reduces seems to be <code>0.95</code> or
* <code>1.75</code> multiplied by (<<i>no. of nodes</i>> *
- * <a href="[EMAIL
PROTECTED]/../hadoop-default.html#mapred.tasktracker.tasks.maximum">
- * mapred.tasktracker.tasks.maximum</a>).
+ * <a href="[EMAIL
PROTECTED]/../hadoop-default.html#mapred.tasktracker.reduce.tasks.maximum">
+ * mapred.tasktracker.reduce.tasks.maximum</a>).
* </p>
*
* <p>With <code>0.95</code> all of the reduces can launch immediately and
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobSubmissionProtocol.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobSubmissionProtocol.java?rev=597057&r1=597056&r2=597057&view=diff
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobSubmissionProtocol.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobSubmissionProtocol.java
Wed Nov 21 04:35:27 2007
@@ -33,8 +33,10 @@
*changed.
*Changed to 4 since killTask(String,boolean) is added
*Version 4: added jobtracker state to ClusterStatus
+ *Version 5: max_tasks in ClusterStatus is replaced by
+ * max_map_tasks and max_reduce_tasks for HADOOP-1274
*/
- public static final long versionID = 4L;
+ public static final long versionID = 5L;
/**
* Allocate a name for the job.
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=597057&r1=597056&r2=597057&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Wed
Nov 21 04:35:27 2007
@@ -506,8 +506,8 @@
private String trackerIdentifier;
long startTime;
int totalSubmissions = 0;
-
- private int totalTaskCapacity;
+ private int totalMapTaskCapacity;
+ private int totalReduceTaskCapacity;
private HostsFileReader hostsReader;
//
@@ -1230,7 +1230,8 @@
if (oldStatus != null) {
totalMaps -= oldStatus.countMapTasks();
totalReduces -= oldStatus.countReduceTasks();
- totalTaskCapacity -= oldStatus.getMaxTasks();
+ totalMapTaskCapacity -= oldStatus.getMaxMapTasks();
+ totalReduceTaskCapacity -= oldStatus.getMaxReduceTasks();
if (status == null) {
taskTrackers.remove(trackerName);
}
@@ -1238,7 +1239,8 @@
if (status != null) {
totalMaps += status.countMapTasks();
totalReduces += status.countReduceTasks();
- totalTaskCapacity += status.getMaxTasks();
+ totalMapTaskCapacity += status.getMaxMapTasks();
+ totalReduceTaskCapacity += status.getMaxReduceTasks();
taskTrackers.put(trackerName, status);
}
return oldStatus != null;
@@ -1320,17 +1322,17 @@
}
}
- int maxCurrentTasks = tts.getMaxTasks();
-
+ int maxCurrentMapTasks = tts.getMaxMapTasks();
+ int maxCurrentReduceTasks = tts.getMaxReduceTasks();
// find out the maximum number of maps or reduces that we are willing
// to run on any node.
int maxMapLoad = 0;
int maxReduceLoad = 0;
if (numTaskTrackers > 0) {
- maxMapLoad = Math.min(maxCurrentTasks,
+ maxMapLoad = Math.min(maxCurrentMapTasks,
(int) Math.ceil((double) remainingMapLoad /
numTaskTrackers));
- maxReduceLoad = Math.min(maxCurrentTasks,
+ maxReduceLoad = Math.min(maxCurrentReduceTasks,
(int) Math.ceil((double) remainingReduceLoad
/ numTaskTrackers));
}
@@ -1380,10 +1382,10 @@
totalNeededMaps += job.desiredMaps();
int padding = 0;
if (numTaskTrackers > MIN_CLUSTER_SIZE_FOR_PADDING) {
- padding = Math.min(maxCurrentTasks,
+ padding = Math.min(maxCurrentMapTasks,
(int)(totalNeededMaps * PAD_FRACTION));
}
- if (totalMaps + padding >= totalTaskCapacity) {
+ if (totalMaps + padding >= totalMapTaskCapacity) {
break;
}
}
@@ -1418,10 +1420,10 @@
int padding = 0;
if (numTaskTrackers > MIN_CLUSTER_SIZE_FOR_PADDING) {
padding =
- Math.min(maxCurrentTasks,
+ Math.min(maxCurrentReduceTasks,
(int) (totalNeededReduces * PAD_FRACTION));
}
- if (totalReduces + padding >= totalTaskCapacity) {
+ if (totalReduces + padding >= totalReduceTaskCapacity) {
break;
}
}
@@ -1577,7 +1579,8 @@
return new ClusterStatus(taskTrackers.size(),
totalMaps,
totalReduces,
- totalTaskCapacity,
+ totalMapTaskCapacity,
+ totalReduceTaskCapacity,
state);
}
}
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java?rev=597057&r1=597056&r2=597057&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
Wed Nov 21 04:35:27 2007
@@ -324,7 +324,7 @@
}
public ClusterStatus getClusterStatus() {
- return new ClusterStatus(1, map_tasks, reduce_tasks, 1,
+ return new ClusterStatus(1, map_tasks, reduce_tasks, 1, 1,
JobTracker.State.RUNNING);
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=597057&r1=597056&r2=597057&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Wed
Nov 21 04:35:27 2007
@@ -152,7 +152,8 @@
private static final String JOBCACHE = "jobcache";
private JobConf fConf;
private MapOutputFile mapOutputFile;
- private int maxCurrentTasks;
+ private int maxCurrentMapTasks;
+ private int maxCurrentReduceTasks;
private int failures;
private int finishedCount[] = new int[1];
private MapEventsFetcherThread mapEventsFetcher;
@@ -259,7 +260,9 @@
if (metricsRecord != null) {
metricsRecord.setMetric("maps_running", mapTotal);
metricsRecord.setMetric("reduces_running", reduceTotal);
- metricsRecord.setMetric("taskSlots", (short)maxCurrentTasks);
+ metricsRecord.setMetric("mapTaskSlots", (short)maxCurrentMapTasks);
+ metricsRecord.setMetric("reduceTaskSlots",
+ (short)maxCurrentReduceTasks);
metricsRecord.incrMetric("tasks_completed", numCompletedTasks);
metricsRecord.incrMetric("tasks_failed_timeout", timedoutTasks);
metricsRecord.incrMetric("tasks_failed_ping", tasksFailedPing);
@@ -414,8 +417,10 @@
this.fConf.get("mapred.task.tracker.report.bindAddress", "127.0.0.1");
// RPC initialization
+ int max = maxCurrentMapTasks > maxCurrentReduceTasks ?
+ maxCurrentMapTasks : maxCurrentReduceTasks;
this.taskReportServer =
- RPC.getServer(this, bindAddress, 0, maxCurrentTasks, false, this.fConf);
+ RPC.getServer(this, bindAddress, 0, max, false, this.fConf);
this.taskReportServer.start();
// get the assigned address
@@ -694,10 +699,45 @@
}
/**
+ * Handles deprecated "mapred.tasktracker.tasks.maximum"
+ * @param newMax new max values specified through
+ * mapred.tasktracker.map.tasks.maximum or
+ * mapred.tasktracker.reduce.tasks.maximum
+ * @param oldMax old max value specified through
+ * mapred.tasktracker.tasks.maximum
+ * @param def default value if max tasks not specified at all.
+ * @return new value supercedes old value. If both new and old values
+ * are not set, default value is returned.
+ */
+ private int handleDeprecatedMaxTasks(String newMax,
+ String oldMax,
+ int def) {
+ try {
+ if (newMax != null) {
+ return Integer.parseInt(newMax);
+ }
+ if (oldMax != null ) {
+ LOG.warn("mapred.tasktracker.tasks.maximum is deprecated. Use " +
+ "mapred.tasktracker.map.tasks.maximum and " +
+ "mapred.tasktracker.reduce.tasks.maximum instead.");
+ return Integer.parseInt(oldMax);
+ }
+ } catch (NumberFormatException ne) {
+ return def;
+ }
+ return def;
+ }
+
+ /**
* Start with the local machine name, and the default JobTracker
*/
public TaskTracker(JobConf conf) throws IOException {
- maxCurrentTasks = conf.getInt("mapred.tasktracker.tasks.maximum", 2);
+ maxCurrentMapTasks = handleDeprecatedMaxTasks(
+ conf.get("mapred.tasktracker.map.tasks.maximum"),
+ conf.get("mapred.tasktracker.tasks.maximum"), 2);
+ maxCurrentReduceTasks = handleDeprecatedMaxTasks(
+ conf.get("mapred.tasktracker.reduce.tasks.maximum"),
+ conf.get("mapred.tasktracker.tasks.maximum"), 2);
this.fConf = conf;
this.jobTrackAddr = JobTracker.getAddress(conf);
this.mapOutputFile = new MapOutputFile();
@@ -868,7 +908,9 @@
synchronized (this) {
status = new TaskTrackerStatus(taskTrackerName, localHostname,
httpPort,
cloneAndResetRunningTaskStatuses(),
- failures, maxCurrentTasks);
+ failures,
+ maxCurrentMapTasks,
+ maxCurrentReduceTasks);
}
} else {
LOG.info("Resending 'status' to '" + jobTrackAddr.getHostName() +
@@ -881,8 +923,8 @@
boolean askForNewTask;
long localMinSpaceStart;
synchronized (this) {
- askForNewTask = (mapTotal < maxCurrentTasks ||
- reduceTotal < maxCurrentTasks) &&
+ askForNewTask = (mapTotal < maxCurrentMapTasks ||
+ reduceTotal < maxCurrentReduceTasks) &&
acceptNewTasks;
localMinSpaceStart = minSpaceStart;
}
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java?rev=597057&r1=597056&r2=597057&view=diff
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java
Wed Nov 21 04:35:27 2007
@@ -46,7 +46,8 @@
List<TaskStatus> taskReports;
volatile long lastSeen;
- int maxTasks;
+ private int maxMapTasks;
+ private int maxReduceTasks;
/**
*/
@@ -58,15 +59,16 @@
*/
public TaskTrackerStatus(String trackerName, String host,
int httpPort, List<TaskStatus> taskReports,
- int failures, int maxTasks) {
+ int failures, int maxMapTasks,
+ int maxReduceTasks) {
this.trackerName = trackerName;
this.host = host;
this.httpPort = httpPort;
this.taskReports = new ArrayList<TaskStatus>(taskReports);
this.failures = failures;
-
- this.maxTasks = maxTasks;
+ this.maxMapTasks = maxMapTasks;
+ this.maxReduceTasks = maxReduceTasks;
}
/**
@@ -158,10 +160,12 @@
* and 1 reduce concurrently).
* @return maximum tasks this node supports
*/
- public int getMaxTasks() {
- return maxTasks;
+ public int getMaxMapTasks() {
+ return maxMapTasks;
}
-
+ public int getMaxReduceTasks() {
+ return maxReduceTasks;
+ }
///////////////////////////////////////////
// Writable
///////////////////////////////////////////
@@ -170,8 +174,8 @@
UTF8.writeString(out, host);
out.writeInt(httpPort);
out.writeInt(failures);
- out.writeInt(maxTasks);
-
+ out.writeInt(maxMapTasks);
+ out.writeInt(maxReduceTasks);
out.writeInt(taskReports.size());
for (TaskStatus taskStatus : taskReports) {
TaskStatus.writeTaskStatus(out, taskStatus);
@@ -183,8 +187,8 @@
this.host = UTF8.readString(in);
this.httpPort = in.readInt();
this.failures = in.readInt();
- this.maxTasks = in.readInt();
-
+ this.maxMapTasks = in.readInt();
+ this.maxReduceTasks = in.readInt();
taskReports.clear();
int numTasks = in.readInt();
for (int i = 0; i < numTasks; i++) {
Modified:
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/SortValidator.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/SortValidator.java?rev=597057&r1=597056&r2=597057&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/SortValidator.java
(original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/SortValidator.java
Wed Nov 21 04:35:27 2007
@@ -472,9 +472,12 @@
jobConf.getInt("test.sortvalidate.maps_per_host", 10);
}
if (noReduces == -1) {
- noReduces = cluster.getTaskTrackers() *
- jobConf.getInt("test.sortvalidate.reduces_per_host",
- cluster.getMaxTasks());
+ noReduces = (int) (cluster.getMaxReduceTasks() * 0.9);
+ String sortReduces = jobConf.get("test.sortvalidate.reduces_per_host");
+ if (sortReduces != null) {
+ noReduces = cluster.getTaskTrackers() *
+ Integer.parseInt(sortReduces);
+ }
}
jobConf.setNumMapTasks(noMaps);
jobConf.setNumReduceTasks(noReduces);
Modified: lucene/hadoop/trunk/src/webapps/job/jobtracker.jsp
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/job/jobtracker.jsp?rev=597057&r1=597056&r2=597057&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/webapps/job/jobtracker.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/job/jobtracker.jsp Wed Nov 21 04:35:27 2007
@@ -68,19 +68,22 @@
JobTracker tracker) throws IOException {
ClusterStatus status = tracker.getClusterStatus();
String tasksPerNode = status.getTaskTrackers() > 0 ?
- percentFormat.format(((double)status.getMaxTasks()) /
status.getTaskTrackers()) :
+ percentFormat.format(((double)(status.getMaxMapTasks() +
+ status.getMaxReduceTasks())) / status.getTaskTrackers()):
"-";
out.print("<table border=\"2\" cellpadding=\"5\" cellspacing=\"2\">\n"+
"<tr><th>Maps</th><th>Reduces</th>" +
"<th>Total Submissions</th>" +
- "<th>Nodes</th><th>Task Capacity</th><th>Avg.
Tasks/Node</th></tr>\n");
+ "<th>Nodes</th><th>Map Task Capacity</th>" +
+ "<th>Reduce Task Capacity</th><th>Avg. Tasks/Node</th></tr>\n");
out.print("<tr><td>" + status.getMapTasks() + "</td><td>" +
status.getReduceTasks() + "</td><td>" +
tracker.getTotalSubmissions() +
"</td><td><a href=\"machines.jsp\">" +
status.getTaskTrackers() +
- "</a></td><td>" + status.getMaxTasks() +
- "</td><td>" + tasksPerNode +
+ "</a></td><td>" + status.getMaxMapTasks() +
+ "</td><td>" + status.getMaxReduceTasks() +
+ "</td><td>" + tasksPerNode +
"</td></tr></table>\n");
}%>
Modified: lucene/hadoop/trunk/src/webapps/job/machines.jsp
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/job/machines.jsp?rev=597057&r1=597056&r2=597057&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/webapps/job/machines.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/job/machines.jsp Wed Nov 21 04:35:27 2007
@@ -25,7 +25,9 @@
out.print("<table border=\"2\" cellpadding=\"5\" cellspacing=\"2\">\n");
out.print("<tr><td align=\"center\" colspan=\"6\"><b>Task
Trackers</b></td></tr>\n");
out.print("<tr><td><b>Name</b></td><td><b>Host</b></td>" +
- "<td><b># running tasks</b></td><td><b>Max Tasks</b></td>" +
+ "<td><b># running tasks</b></td>" +
+ "<td><b>Max Map Tasks</b></td>" +
+ "<td><b>Max Reduce Tasks</b></td>" +
"<td><b>Failures</b></td>" +
"<td><b>Seconds since heartbeat</b></td></tr>\n");
int maxFailures = 0;
@@ -50,7 +52,8 @@
out.print(tt.getHost() + ":" + tt.getHttpPort() + "/\">");
out.print(tt.getTrackerName() + "</a></td><td>");
out.print(tt.getHost() + "</td><td>" + numCurTasks +
- "</td><td>" + tt.getMaxTasks() +
+ "</td><td>" + tt.getMaxMapTasks() +
+ "</td><td>" + tt.getMaxReduceTasks() +
"</td><td>" + numFailures +
"</td><td>" + sinceHeartbeat + "</td></tr>\n");
}
