Author: omalley
Date: Wed Jul 30 15:16:45 2008
New Revision: 681223
URL: http://svn.apache.org/viewvc?rev=681223&view=rev
Log:
HADOOP-3848. Cache calls to getSystemDir in the TaskTracker instead of
calling it for each task start. Contributed by Arun C. Murthy.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=681223&r1=681222&r2=681223&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Jul 30 15:16:45 2008
@@ -190,6 +190,9 @@
HADOOP-3819. Unset LANG and LC_CTYPE in saveVersion.sh to make it
compatible with non-English locales. (Rong-En Fan via cdouglas)
+ HADOOP-3848. Cache calls to getSystemDir in the TaskTracker instead of
+ calling it for each task start. (acmurthy via omalley)
+
Release 0.18.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=681223&r1=681222&r2=681223&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
(original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java Wed
Jul 30 15:16:45 2008
@@ -121,7 +121,13 @@
* again.
*/
TaskTrackerStatus status = null;
-
+
+ // The system-directory on HDFS where job files are stored
+ Path systemDirectory = null;
+
+ // The filesystem where job files are stored
+ FileSystem systemFS = null;
+
StatusHttpServer server = null;
volatile boolean shuttingDown = false;
@@ -633,12 +639,10 @@
Path jobFile = new Path(t.getJobFile());
// Get sizes of JobFile and JarFile
// sizes are -1 if they are not present.
- Path systemDir = new Path(jobClient.getSystemDir());
- FileSystem fs = systemDir.getFileSystem(fConf);
FileStatus status = null;
long jobFileSize = -1;
try {
- status = fs.getFileStatus(jobFile);
+ status = systemFS.getFileStatus(jobFile);
jobFileSize = status.getLen();
} catch(FileNotFoundException fe) {
jobFileSize = -1;
@@ -664,7 +668,7 @@
throw new IOException("Not able to create job directory "
+ jobDir.toString());
}
- fs.copyToLocalFile(jobFile, localJobFile);
+ systemFS.copyToLocalFile(jobFile, localJobFile);
JobConf localJobConf = new JobConf(localJobFile);
// create the 'work' directory
@@ -685,7 +689,7 @@
if (jarFile != null) {
Path jarFilePath = new Path(jarFile);
try {
- status = fs.getFileStatus(jarFilePath);
+ status = systemFS.getFileStatus(jarFilePath);
jarFileSize = status.getLen();
} catch(FileNotFoundException fe) {
jarFileSize = -1;
@@ -700,7 +704,7 @@
if (!localFs.mkdirs(localJarFile.getParent())) {
throw new IOException("Mkdirs failed to create jars directory ");
}
- fs.copyToLocalFile(jarFilePath, localJarFile);
+ systemFS.copyToLocalFile(jarFilePath, localJarFile);
localJobConf.setJar(localJarFile.toString());
OutputStream out = localFs.create(localJobFile);
try {
@@ -906,7 +910,9 @@
}
}
- //verify the buildVersion if justStarted
+ // If the TaskTracker is just starting up:
+ // 1. Verify the buildVersion
+ // 2. Get the system directory & filesystem
if(justStarted){
String jobTrackerBV = jobClient.getBuildVersion();
if(!VersionInfo.getBuildVersion().equals(jobTrackerBV)) {
@@ -921,6 +927,13 @@
}
return State.DENIED;
}
+
+ String dir = jobClient.getSystemDir();
+ if (dir == null) {
+ throw new IOException("Failed to get system directory");
+ }
+ systemDirectory = new Path(dir);
+ systemFS = systemDirectory.getFileSystem(fConf);
}
// Send the heartbeat and process the jobtracker's directives
