Author: ddas
Date: Fri Feb 20 13:17:20 2009
New Revision: 746228
URL: http://svn.apache.org/viewvc?rev=746228&view=rev
Log:
Merge -r 746226:746227 from trunk onto 0.20. Fixes HADOOP-5233.
Modified:
hadoop/core/branches/branch-0.20/ (props changed)
hadoop/core/branches/branch-0.20/CHANGES.txt (contents, props changed)
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskStatus.java
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
Propchange: hadoop/core/branches/branch-0.20/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Feb 20 13:17:20 2009
@@ -1,2 +1,2 @@
/hadoop/core/branches/branch-0.19:713112
-/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206
+/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227
Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=746228&r1=746227&r2=746228&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Fri Feb 20 13:17:20 2009
@@ -620,6 +620,11 @@
Scheduler accesses the tasktrackers stored by the JobTracker.
(Rahul Kumar Singh via yhemanth)
+ HADOOP-5233. Addresses the three issues - Race condition in updating
+ status, NPE in TaskTracker task localization when the conf file is missing
+ (HADOOP-5234) and NPE in handling KillTaskAction of a cleanup task
(HADOOP-5235).
+ (Amareshwari Sriramadasu via ddas)
+
Release 0.19.1 - Unreleased
IMPROVEMENTS
Propchange: hadoop/core/branches/branch-0.20/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Feb 20 13:17:20 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
-/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206
+/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227
Modified:
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=746228&r1=746227&r2=746228&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Fri Feb 20 13:17:20 2009
@@ -815,6 +815,8 @@
} else {
reduceCleanupTasks.add(taskid);
}
+ // Remove the task entry from jobtracker
+ jobtracker.removeTaskEntry(taskid);
}
//For a failed task update the JT datastructures.
else if (state == TaskStatus.State.FAILED ||
Modified:
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java?rev=746228&r1=746227&r2=746228&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java
(original)
+++
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java
Fri Feb 20 13:17:20 2009
@@ -514,8 +514,7 @@
if (sendProgress) {
// we need to send progress update
updateCounters();
- taskStatus.statusUpdate(getState(),
- taskProgress.get(),
+ taskStatus.statusUpdate(taskProgress.get(),
taskProgress.toString(),
counters);
taskFound = umbilical.statusUpdate(taskId, taskStatus);
@@ -702,8 +701,7 @@
private void sendLastUpdate(TaskUmbilicalProtocol umbilical)
throws IOException {
// send a final status report
- taskStatus.statusUpdate(getState(),
- taskProgress.get(),
+ taskStatus.statusUpdate(taskProgress.get(),
taskProgress.toString(),
counters);
statusUpdate(umbilical);
Modified:
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java?rev=746228&r1=746227&r2=746228&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java
(original)
+++
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java
Fri Feb 20 13:17:20 2009
@@ -915,7 +915,6 @@
t.setTaskCleanupTask();
t.setState(taskStatuses.get(taskid).getRunState());
cleanupTasks.put(taskid, taskTracker);
- jobtracker.removeTaskEntry(taskid);
}
t.setConf(conf);
LOG.debug("Launching task with skipRanges:"+failedRanges.getSkipRanges());
Modified:
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskStatus.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskStatus.java?rev=746228&r1=746227&r2=746228&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskStatus.java
(original)
+++
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskStatus.java
Fri Feb 20 13:17:20 2009
@@ -54,7 +54,7 @@
private long finishTime;
private long outputSize;
- private Phase phase = Phase.STARTING;
+ private volatile Phase phase = Phase.STARTING;
private Counters counters;
private boolean includeCounters;
private SortedRanges.Range nextRecordRange = new SortedRanges.Range();
@@ -267,16 +267,15 @@
/**
* Update the status of the task.
*
- * @param runstate
+ * This update is done by ping thread before sending the status.
+ *
* @param progress
* @param state
* @param counters
*/
- synchronized void statusUpdate(State runState,
- float progress,
+ synchronized void statusUpdate(float progress,
String state,
Counters counters) {
- setRunState(runState);
setProgress(progress);
setStateString(state);
setCounters(counters);
Modified:
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=746228&r1=746227&r2=746228&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
(original)
+++
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
Fri Feb 20 13:17:20 2009
@@ -2020,10 +2020,16 @@
if (this.done ||
(this.taskStatus.getRunState() != TaskStatus.State.RUNNING &&
this.taskStatus.getRunState() != TaskStatus.State.COMMIT_PENDING &&
- !isCleaningup())) {
+ !isCleaningup()) ||
+ ((this.taskStatus.getRunState() == TaskStatus.State.COMMIT_PENDING ||
+ this.taskStatus.getRunState() == TaskStatus.State.FAILED_UNCLEAN ||
+ this.taskStatus.getRunState() == TaskStatus.State.KILLED_UNCLEAN) &&
+ taskStatus.getRunState() == TaskStatus.State.RUNNING)) {
//make sure we ignore progress messages after a task has
//invoked TaskUmbilicalProtocol.done() or if the task has been
- //KILLED/FAILED
+ //KILLED/FAILED/FAILED_UNCLEAN/KILLED_UNCLEAN
+ //Also ignore progress update if the state change is from
+ //COMMIT_PENDING/FAILED_UNCLEAN/KILLED_UNCLEA to RUNNING
LOG.info(task.getTaskID() + " Ignoring status-update since " +
((this.done) ? "task is 'done'" :
("runState: " + this.taskStatus.getRunState()))
@@ -2376,7 +2382,10 @@
if (wasFailure) {
failures += 1;
}
- runner.kill();
+ // runner could be null if task-cleanup attempt is not localized yet
+ if (runner != null) {
+ runner.kill();
+ }
setTaskFailState(wasFailure);
} else if (taskStatus.getRunState() == TaskStatus.State.UNASSIGNED) {
if (wasFailure) {
@@ -2455,6 +2464,11 @@
}
synchronized (this) {
try {
+ // localJobConf could be null if localization has not happened
+ // then no cleanup will be required.
+ if (localJobConf == null) {
+ return;
+ }
String taskDir = getLocalTaskDir(task.getJobID().toString(),
taskId.toString(), task.isTaskCleanupTask());
if (needCleanup) {
@@ -2591,7 +2605,8 @@
public synchronized void commitPending(TaskAttemptID taskid,
TaskStatus taskStatus)
throws IOException {
- LOG.info("Task " + taskid + " is in COMMIT_PENDING");
+ LOG.info("Task " + taskid + " is in commit-pending," +"" +
+ " task state:" +taskStatus.getRunState());
statusUpdate(taskid, taskStatus);
reportTaskFinished(taskid, true);
}
