Author: yhemanth
Date: Wed Feb 25 14:59:24 2009
New Revision: 747805
URL: http://svn.apache.org/viewvc?rev=747805&view=rev
Log:
Merge -r 747801:747802 from trunk to branch 0.20 to fix HADOOP-5154.
Modified:
hadoop/core/branches/branch-0.20/ (props changed)
hadoop/core/branches/branch-0.20/CHANGES.txt (contents, props changed)
hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
Propchange: hadoop/core/branches/branch-0.20/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 25 14:59:24 2009
@@ -1,2 +1,2 @@
/hadoop/core/branches/branch-0.19:713112
-/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746902-746903,746944,746968,746970,747279
+/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746902-746903,746944,746968,746970,747279,747802
Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=747805&r1=747804&r2=747805&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Wed Feb 25 14:59:24 2009
@@ -661,6 +661,13 @@
HADOOP-5272. Fixes a problem to do with detecting whether an attempt is
the first
attempt of a Task. This affects JobTracker restart. (Amar Kamat via ddas)
+Release 0.19.2 - Unreleased
+
+ BUG FIXES
+
+ HADOOP-5154. Fixes a deadlock in the fairshare scheduler.
+ (Matei Zaharia via yhemanth)
+
Release 0.19.1 - Unreleased
IMPROVEMENTS
Propchange: hadoop/core/branches/branch-0.20/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 25 14:59:24 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
-/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746944,746968,746970,747279
+/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746944,746968,746970,747279,747802
Modified:
hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java?rev=747805&r1=747804&r2=747805&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
(original)
+++
hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
Wed Feb 25 14:59:24 2009
@@ -708,11 +708,11 @@
clusterStatus.getMaxMapTasks() : clusterStatus.getMaxReduceTasks());
}
- public boolean getUseFifo() {
+ public synchronized boolean getUseFifo() {
return useFifo;
}
- public void setUseFifo(boolean useFifo) {
+ public synchronized void setUseFifo(boolean useFifo) {
this.useFifo = useFifo;
}
Modified:
hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java?rev=747805&r1=747804&r2=747805&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
(original)
+++
hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
Wed Feb 25 14:59:24 2009
@@ -86,11 +86,11 @@
return;
}
if (request.getParameter("setPool") != null) {
- PoolManager poolMgr = scheduler.getPoolManager();
- synchronized (poolMgr) {
+ Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+ synchronized (scheduler) {
+ PoolManager poolMgr = scheduler.getPoolManager();
String pool = request.getParameter("setPool");
String jobId = request.getParameter("jobid");
- Collection<JobInProgress> runningJobs = jobTracker.runningJobs();
for (JobInProgress job: runningJobs) {
if (job.getProfile().getJobID().toString().equals(jobId)) {
poolMgr.setPool(job, pool);
@@ -103,12 +103,11 @@
return;
}
if (request.getParameter("setPriority") != null) {
- PoolManager poolMgr = scheduler.getPoolManager();
- synchronized (poolMgr) {
+ Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+ synchronized (scheduler) {
JobPriority priority = JobPriority.valueOf(request.getParameter(
"setPriority"));
String jobId = request.getParameter("jobid");
- Collection<JobInProgress> runningJobs = jobTracker.runningJobs();
for (JobInProgress job: runningJobs) {
if (job.getProfile().getJobID().toString().equals(jobId)) {
job.setPriority(priority);
@@ -127,8 +126,6 @@
jobTracker.getJobTrackerMachine());
out.print("<html><head>");
out.printf("<title>%s Job Scheduler Admininstration</title>\n", hostname);
- out.printf("<META http-equiv=\"refresh\" content=\"15;URL=/scheduler%s\">",
- advancedView ? "?advanced" : "");
out.print("<link rel=\"stylesheet\" type=\"text/css\" " +
"href=\"/static/hadoop.css\">\n");
out.print("</head><body>\n");
@@ -145,8 +142,8 @@
* Print a view of pools to the given output writer.
*/
private void showPools(PrintWriter out, boolean advancedView) {
- PoolManager poolManager = scheduler.getPoolManager();
- synchronized(poolManager) {
+ synchronized(scheduler) {
+ PoolManager poolManager = scheduler.getPoolManager();
out.print("<h2>Pools</h2>\n");
out.print("<table border=\"2\" cellpadding=\"5\" cellspacing=\"2\">\n");
out.print("<tr><th>Pool</th><th>Running Jobs</th>" +
@@ -208,50 +205,52 @@
out.print("<th>Finished</th><th>Running</th><th>Fair Share</th>" +
(advancedView ? "<th>Weight</th><th>Deficit</th><th>minReduces</th>" :
""));
out.print("</tr>\n");
- Collection<JobInProgress> runningJobs = jobTracker.runningJobs();
- for (JobInProgress job: runningJobs) {
- JobProfile profile = job.getProfile();
- JobInfo info = scheduler.infos.get(job);
- if (info == null) { // Job finished, but let's show 0's for info
- info = new JobInfo();
- }
- out.print("<tr>\n");
- out.printf("<td>%s</td>\n", DATE_FORMAT.format(
- new Date(job.getStartTime())));
- out.printf("<td><a href=\"jobdetails.jsp?jobid=%s\">%s</a></td>",
- profile.getJobID(), profile.getJobID());
- out.printf("<td>%s</td>\n", profile.getUser());
- out.printf("<td>%s</td>\n", profile.getJobName());
- out.printf("<td>%s</td>\n", generateSelect(
- scheduler.getPoolManager().getPoolNames(),
- scheduler.getPoolManager().getPoolName(job),
- "/scheduler?setPool=<CHOICE>&jobid=" + profile.getJobID() +
- (advancedView ? "&advanced" : "")));
- out.printf("<td>%s</td>\n", generateSelect(
- Arrays.asList(new String[]
- {"VERY_LOW", "LOW", "NORMAL", "HIGH", "VERY_HIGH"}),
- job.getPriority().toString(),
- "/scheduler?setPriority=<CHOICE>&jobid=" + profile.getJobID() +
- (advancedView ? "&advanced" : "")));
- out.printf("<td>%d / %d</td><td>%d</td><td>%8.1f</td>\n",
- job.finishedMaps(), job.desiredMaps(), info.runningMaps,
- info.mapFairShare);
- if (advancedView) {
- out.printf("<td>%8.1f</td>\n", info.mapWeight);
- out.printf("<td>%s</td>\n", info.neededMaps > 0 ?
- (info.mapDeficit / 1000) + "s" : "--");
- out.printf("<td>%d</td>\n", info.minMaps);
- }
- out.printf("<td>%d / %d</td><td>%d</td><td>%8.1f</td>\n",
- job.finishedReduces(), job.desiredReduces(), info.runningReduces,
- info.reduceFairShare);
- if (advancedView) {
- out.printf("<td>%8.1f</td>\n", info.reduceWeight);
- out.printf("<td>%s</td>\n", info.neededReduces > 0 ?
- (info.reduceDeficit / 1000) + "s" : "--");
- out.printf("<td>%d</td>\n", info.minReduces);
+ Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+ synchronized (scheduler) {
+ for (JobInProgress job: runningJobs) {
+ JobProfile profile = job.getProfile();
+ JobInfo info = scheduler.infos.get(job);
+ if (info == null) { // Job finished, but let's show 0's for info
+ info = new JobInfo();
+ }
+ out.print("<tr>\n");
+ out.printf("<td>%s</td>\n", DATE_FORMAT.format(
+ new Date(job.getStartTime())));
+ out.printf("<td><a href=\"jobdetails.jsp?jobid=%s\">%s</a></td>",
+ profile.getJobID(), profile.getJobID());
+ out.printf("<td>%s</td>\n", profile.getUser());
+ out.printf("<td>%s</td>\n", profile.getJobName());
+ out.printf("<td>%s</td>\n", generateSelect(
+ scheduler.getPoolManager().getPoolNames(),
+ scheduler.getPoolManager().getPoolName(job),
+ "/scheduler?setPool=<CHOICE>&jobid=" + profile.getJobID() +
+ (advancedView ? "&advanced" : "")));
+ out.printf("<td>%s</td>\n", generateSelect(
+ Arrays.asList(new String[]
+ {"VERY_LOW", "LOW", "NORMAL", "HIGH", "VERY_HIGH"}),
+ job.getPriority().toString(),
+ "/scheduler?setPriority=<CHOICE>&jobid=" + profile.getJobID() +
+ (advancedView ? "&advanced" : "")));
+ out.printf("<td>%d / %d</td><td>%d</td><td>%8.1f</td>\n",
+ job.finishedMaps(), job.desiredMaps(), info.runningMaps,
+ info.mapFairShare);
+ if (advancedView) {
+ out.printf("<td>%8.1f</td>\n", info.mapWeight);
+ out.printf("<td>%s</td>\n", info.neededMaps > 0 ?
+ (info.mapDeficit / 1000) + "s" : "--");
+ out.printf("<td>%d</td>\n", info.minMaps);
+ }
+ out.printf("<td>%d / %d</td><td>%d</td><td>%8.1f</td>\n",
+ job.finishedReduces(), job.desiredReduces(), info.runningReduces,
+ info.reduceFairShare);
+ if (advancedView) {
+ out.printf("<td>%8.1f</td>\n", info.reduceWeight);
+ out.printf("<td>%s</td>\n", info.neededReduces > 0 ?
+ (info.reduceDeficit / 1000) + "s" : "--");
+ out.printf("<td>%d</td>\n", info.minReduces);
+ }
+ out.print("</tr>\n");
}
- out.print("</tr>\n");
}
out.print("</table>\n");
}
