Author: sharad
Date: Fri Jun 26 12:04:11 2009
New Revision: 788666
URL: http://svn.apache.org/viewvc?rev=788666&view=rev
Log:
MAPREDUCE-657. Fix hardcoded filesystem problem in CompletedJobStatusStore.
Contributed by Amar Kamat.
Modified:
hadoop/common/branches/branch-0.20/CHANGES.txt
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/CompletedJobStatusStore.java
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobStatusPersistency.java
Modified: hadoop/common/branches/branch-0.20/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/CHANGES.txt?rev=788666&r1=788665&r2=788666&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20/CHANGES.txt Fri Jun 26 12:04:11 2009
@@ -150,6 +150,9 @@
MAPREDUCE-130. Delete the jobconf copy from the log directory of the
JobTracker when the job is retired. (Amar Kamat via sharad)
+ MAPREDUCE-657. Fix hardcoded filesystem problem in CompletedJobStatusStore.
+ (Amar Kamat via sharad)
+
Release 0.20.0 - 2009-04-15
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/CompletedJobStatusStore.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/CompletedJobStatusStore.java?rev=788666&r1=788665&r2=788666&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/CompletedJobStatusStore.java
(original)
+++
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/CompletedJobStatusStore.java
Fri Jun 26 12:04:11 2009
@@ -51,12 +51,11 @@
private static long HOUR = 1000 * 60 * 60;
private static long SLEEP_TIME = 1 * HOUR;
- CompletedJobStatusStore(Configuration conf, FileSystem fs) throws
IOException {
+ CompletedJobStatusStore(Configuration conf) throws IOException {
active =
conf.getBoolean("mapred.job.tracker.persist.jobstatus.active", false);
if (active) {
- this.fs = fs;
retainTime =
conf.getInt("mapred.job.tracker.persist.jobstatus.hours", 0) * HOUR;
@@ -64,6 +63,9 @@
conf.get("mapred.job.tracker.persist.jobstatus.dir",
JOB_INFO_STORE_DIR);
Path path = new Path(jobInfoDir);
+
+ // set the fs
+ this.fs = path.getFileSystem(conf);
if (!fs.exists(path)) {
fs.mkdirs(path);
}
@@ -72,6 +74,10 @@
// as retain time is zero, all stored jobstatuses are deleted.
deleteJobStatusDirs();
}
+ LOG.info("Completed job store activated/configured with retain-time : "
+ + retainTime + " , job-info-dir : " + jobInfoDir);
+ } else {
+ LOG.info("Completed job store is inactive");
}
}
Modified:
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=788666&r1=788665&r2=788666&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
(original)
+++
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
Fri Jun 26 12:04:11 2009
@@ -1711,7 +1711,7 @@
NetworkTopology.DEFAULT_HOST_LEVEL);
//initializes the job status store
- completedJobStatusStore = new CompletedJobStatusStore(conf,fs);
+ completedJobStatusStore = new CompletedJobStatusStore(conf);
}
private static SimpleDateFormat getDateFormat() {
Modified:
hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobStatusPersistency.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobStatusPersistency.java?rev=788666&r1=788665&r2=788666&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobStatusPersistency.java
(original)
+++
hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobStatusPersistency.java
Fri Jun 26 12:04:11 2009
@@ -22,11 +22,16 @@
import java.io.Writer;
import java.util.Properties;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
public class TestJobStatusPersistency extends ClusterMapReduceTestCase {
+ static final Path TEST_DIR =
+ new Path(System.getProperty("test.build.data","/tmp"),
+ "job-status-persistence");
+
private JobID runJob() throws Exception {
OutputStream os = getFileSystem().create(new Path(getInputDir(),
"text.txt"));
Writer wr = new OutputStreamWriter(os);
@@ -103,4 +108,29 @@
}
}
+ /**
+ * Test if the completed job status is persisted to localfs.
+ */
+ public void testLocalPersistency() throws Exception {
+ FileSystem fs = FileSystem.getLocal(createJobConf());
+
+ fs.delete(TEST_DIR, true);
+
+ Properties config = new Properties();
+ config.setProperty("mapred.job.tracker.persist.jobstatus.active", "true");
+ config.setProperty("mapred.job.tracker.persist.jobstatus.hours", "1");
+ config.setProperty("mapred.job.tracker.persist.jobstatus.dir",
+ fs.makeQualified(TEST_DIR).toString());
+ stopCluster();
+ startCluster(false, config);
+ JobID jobId = runJob();
+ JobClient jc = new JobClient(createJobConf());
+ RunningJob rj = jc.getJob(jobId);
+ assertNotNull(rj);
+
+ // check if the local fs has the data
+ Path jobInfo = new Path(TEST_DIR, rj.getID() + ".info");
+ assertTrue("Missing job info from the local fs", fs.exists(jobInfo));
+ fs.delete(TEST_DIR, true);
+ }
}
