Author: cutting
Date: Thu May 17 12:59:14 2007
New Revision: 539095
URL: http://svn.apache.org/viewvc?view=rev&rev=539095
Log:
Merge -r 539088:539093 from trunk to 0.13 branch. Fixes: HADOOP-1356,
HADOOP-1363, and HADOOP-1368.
Modified:
lucene/hadoop/branches/branch-0.13/CHANGES.txt
lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobClient.java
lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobInProgress.java
lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobStatus.java
lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/lib/aggregate/ValueHistogram.java
lucene/hadoop/branches/branch-0.13/src/test/org/apache/hadoop/mapred/TestAggregates.java
Modified: lucene/hadoop/branches/branch-0.13/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.13/CHANGES.txt?view=diff&rev=539095&r1=539094&r2=539095
==============================================================================
--- lucene/hadoop/branches/branch-0.13/CHANGES.txt (original)
+++ lucene/hadoop/branches/branch-0.13/CHANGES.txt Thu May 17 12:59:14 2007
@@ -390,6 +390,14 @@
116. HADOOP-1358. Fix a potential bug when DFSClient calls skipBytes.
(Hairong Kuang via cutting)
+117. HADOOP-1356. Fix a bug in ValueHistogram. (Runping Qi via cutting)
+
+118. HADOOP-1363. Fix locking bug in JobClient#waitForCompletion().
+ (omalley via cutting)
+
+119. HADOOP-1368. Fix inconsistent synchronization in JobInProgress.
+ (omalley via cutting)
+
Release 0.12.3 - 2007-04-06
Modified:
lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobClient.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobClient.java?view=diff&rev=539095&r1=539094&r2=539095
==============================================================================
---
lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobClient.java
(original)
+++
lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobClient.java
Thu May 17 12:59:14 2007
@@ -137,7 +137,7 @@
/**
* Blocks until the job is finished
*/
- public synchronized void waitForCompletion() throws IOException {
+ public void waitForCompletion() throws IOException {
while (!isComplete()) {
try {
Thread.sleep(5000);
Modified:
lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobInProgress.java?view=diff&rev=539095&r1=539094&r2=539095
==============================================================================
---
lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++
lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobInProgress.java
Thu May 17 12:59:14 2007
@@ -220,7 +220,9 @@
// Finished time need to be setted here to prevent this job to be retired
// from the job tracker jobs at the next retire iteration.
this.finishTime = System.currentTimeMillis();
- this.status = new JobStatus(status.getJobId(), 1.0f, 1.0f,
JobStatus.SUCCEEDED);
+ status.setMapProgress(1.0f);
+ status.setReduceProgress(1.0f);
+ status.setRunState(JobStatus.SUCCEEDED);
tasksInited = true;
// Special case because the Job is not queued
@@ -263,7 +265,7 @@
public int desiredMaps() {
return numMapTasks;
}
- public int finishedMaps() {
+ public synchronized int finishedMaps() {
return finishedMapTasks;
}
public int desiredReduces() {
@@ -275,7 +277,7 @@
public synchronized int runningReduces() {
return runningReduceTasks;
}
- public int finishedReduces() {
+ public synchronized int finishedReduces() {
return finishedReduceTasks;
}
@@ -485,8 +487,9 @@
/**
* Return a MapTask, if appropriate, to run on the given tasktracker
*/
- public Task obtainNewMapTask(TaskTrackerStatus tts, int clusterSize
- ) throws IOException {
+ public synchronized Task obtainNewMapTask(TaskTrackerStatus tts,
+ int clusterSize
+ ) throws IOException {
if (!tasksInited) {
LOG.info("Cannot create task split for " + profile.getJobId());
return null;
@@ -513,8 +516,9 @@
* We don't have cache-sensitivity for reduce tasks, as they
* work on temporary MapRed files.
*/
- public Task obtainNewReduceTask(TaskTrackerStatus tts,
- int clusterSize) throws IOException {
+ public synchronized Task obtainNewReduceTask(TaskTrackerStatus tts,
+ int clusterSize
+ ) throws IOException {
if (!tasksInited) {
LOG.info("Cannot create task split for " + profile.getJobId());
return null;
Modified:
lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobStatus.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobStatus.java?view=diff&rev=539095&r1=539094&r2=539095
==============================================================================
---
lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobStatus.java
(original)
+++
lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobStatus.java
Thu May 17 12:59:14 2007
@@ -30,8 +30,6 @@
* Describes the current status of a job. This is
* not intended to be a comprehensive piece of data.
* For that, look at JobProfile.
- *
- * @author Mike Cafarella
**************************************************/
public class JobStatus implements Writable {
@@ -83,39 +81,38 @@
/**
* @return Percentage of progress in maps
*/
- public float mapProgress() { return mapProgress; }
+ public synchronized float mapProgress() { return mapProgress; }
/**
* Sets the map progress of this job
* @param p The value of map progress to set to
*/
- void setMapProgress(float p) {
+ synchronized void setMapProgress(float p) {
this.mapProgress = (float) Math.min(1.0, Math.max(0.0, p));
-
}
/**
* @return Percentage of progress in reduce
*/
- public float reduceProgress() { return reduceProgress; }
+ public synchronized float reduceProgress() { return reduceProgress; }
/**
* Sets the reduce progress of this Job
* @param p The value of reduce progress to set to
*/
- void setReduceProgress(float p) {
+ synchronized void setReduceProgress(float p) {
this.reduceProgress = (float) Math.min(1.0, Math.max(0.0, p));
}
/**
* @return running state of the job
*/
- public int getRunState() { return runState; }
+ public synchronized int getRunState() { return runState; }
/**
* Change the current run state of the job.
*/
- public void setRunState(int state) {
+ public synchronized void setRunState(int state) {
this.runState = state;
}
@@ -123,22 +120,22 @@
* Set the start time of the job
* @param startTime The startTime of the job
*/
- void setStartTime(long startTime) { this.startTime = startTime;};
+ synchronized void setStartTime(long startTime) { this.startTime = startTime;}
/**
* @return start time of the job
*/
- public long getStartTime() { return startTime;};
+ synchronized public long getStartTime() { return startTime;}
/**
* @param user The username of the job
*/
- void setUsername(String userName) { this.user = userName;};
+ synchronized void setUsername(String userName) { this.user = userName;}
/**
* @return the username of the job
*/
- public String getUsername() { return this.user;};
+ public synchronized String getUsername() { return this.user;}
///////////////////////////////////////
// Writable
Modified:
lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/lib/aggregate/ValueHistogram.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/lib/aggregate/ValueHistogram.java?view=diff&rev=539095&r1=539094&r2=539095
==============================================================================
---
lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/lib/aggregate/ValueHistogram.java
(original)
+++
lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/lib/aggregate/ValueHistogram.java
Thu May 17 12:59:14 2007
@@ -51,7 +51,7 @@
String valStr = valCountStr;
String countStr = "1";
if (pos >= 0) {
- valCountStr.substring(0, pos);
+ valStr = valCountStr.substring(0, pos);
countStr = valCountStr.substring(pos + 1);
}
Modified:
lucene/hadoop/branches/branch-0.13/src/test/org/apache/hadoop/mapred/TestAggregates.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.13/src/test/org/apache/hadoop/mapred/TestAggregates.java?view=diff&rev=539095&r1=539094&r2=539095
==============================================================================
---
lucene/hadoop/branches/branch-0.13/src/test/org/apache/hadoop/mapred/TestAggregates.java
(original)
+++
lucene/hadoop/branches/branch-0.13/src/test/org/apache/hadoop/mapred/TestAggregates.java
Thu May 17 12:59:14 2007
@@ -106,6 +106,8 @@
boolean success = true;
Path outPath = new Path(OUTPUT_DIR, "part-00000");
String outdata = TestMiniMRWithDFS.readOutput(outPath,job);
+ System.out.println("full out data:");
+ System.out.println(outdata.toString());
outdata = outdata.substring(0, expectedOutput.toString().length());
assertEquals(expectedOutput.toString(),outdata);
