Author: ddas
Date: Mon Sep 29 01:15:59 2008
New Revision: 700024
URL: http://svn.apache.org/viewvc?rev=700024&view=rev
Log:
Merge -r 700020:700021 from trunk onto 0.19 branch. Fixes HADOOP-4189.
Modified:
hadoop/core/branches/branch-0.19/CHANGES.txt
hadoop/core/branches/branch-0.19/conf/hadoop-default.xml
hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java
hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobHistory.java
hadoop/core/branches/branch-0.19/src/webapps/job/loadhistory.jsp
Modified: hadoop/core/branches/branch-0.19/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/CHANGES.txt?rev=700024&r1=700023&r2=700024&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.19/CHANGES.txt Mon Sep 29 01:15:59 2008
@@ -755,6 +755,9 @@
memory management in TaskTracker are disabled on Windows.
(Vinod K V via rangadi)
+ HADOOP-4189. Fixes the history blocksize & intertracker protocol version
+ issues introduced as part of HADOOP-3245. (Amar Kamat via ddas)
+
Release 0.18.2 - Unreleased
BUG FIXES
Modified: hadoop/core/branches/branch-0.19/conf/hadoop-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/conf/hadoop-default.xml?rev=700024&r1=700023&r2=700024&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/conf/hadoop-default.xml (original)
+++ hadoop/core/branches/branch-0.19/conf/hadoop-default.xml Mon Sep 29
01:15:59 2008
@@ -762,19 +762,11 @@
<property>
<name>mapred.jobtracker.job.history.block.size</name>
- <value>0</value>
+ <value>3145728></value>
<description>The block size of the job history file. Since the job recovery
uses job history, its important to dump job history to disk as
- soon as possible.
- </description>
-</property>
-
-<property>
- <name>mapred.jobtracker.job.history.buffer.size</name>
- <value>4096</value>
- <description>The buffer size for the job history file. Since the job
- recovery uses job history, its important to frequently flush
the
- job history to disk. This will minimize the loss in recovery.
+ soon as possible. Note that this is an expert level parameter.
+ The default value is set to 3 MB.
</description>
</property>
Modified:
hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java?rev=700024&r1=700023&r2=700024&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java
(original)
+++
hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java
Mon Sep 29 01:15:59 2008
@@ -48,6 +48,8 @@
* Version 16: adds ResourceStatus to TaskTrackerStatus for HADOOP-3759
* Version 17: Changed format of Task and TaskStatus for HADOOP-3150
* Version 18: Changed status message due to changes in TaskStatus
+ * Changed heartbeat to piggyback JobTracker restart information
+ so that the TaskTracker can synchronize itself.
*/
public static final long versionID = 18L;
Modified:
hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=700024&r1=700023&r2=700024&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobHistory.java
(original)
+++
hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobHistory.java
Mon Sep 29 01:15:59 2008
@@ -85,7 +85,6 @@
private static boolean disableHistory = false;
private static final String SECONDARY_FILE_SUFFIX = ".recover";
private static long jobHistoryBlockSize = 0;
- private static int jobHistoryBufferSize = 0;
private static String jobtrackerHostname;
/**
* Record types are identifiers for each line of log in history files.
@@ -148,14 +147,10 @@
}
conf.set("hadoop.job.history.location", LOG_DIR);
disableHistory = false;
- // set the job history block size
+ // set the job history block size (default is 3MB)
jobHistoryBlockSize =
conf.getLong("mapred.jobtracker.job.history.block.size",
- fs.getDefaultBlockSize());
- // set the job history buffer size
- jobHistoryBufferSize =
- Integer.parseInt(conf.get("mapred.jobtracker.job.history.buffer.size",
- "4096"));
+ 3 * 1024 * 1024);
} catch(IOException e) {
LOG.error("Failed to initialize JobHistory log file", e);
disableHistory = true;
@@ -729,8 +724,10 @@
logFile = recoverJobHistoryFile(jobConf, logFile);
+ int defaultBufferSize =
+ fs.getConf().getInt("io.file.buffer.size", 4096);
out = fs.create(logFile, FsPermission.getDefault(), true,
- jobHistoryBufferSize,
+ defaultBufferSize,
fs.getDefaultReplication(),
jobHistoryBlockSize, null);
writer = new PrintWriter(out);
Modified: hadoop/core/branches/branch-0.19/src/webapps/job/loadhistory.jsp
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/webapps/job/loadhistory.jsp?rev=700024&r1=700023&r2=700024&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/webapps/job/loadhistory.jsp (original)
+++ hadoop/core/branches/branch-0.19/src/webapps/job/loadhistory.jsp Mon Sep 29
01:15:59 2008
@@ -24,9 +24,16 @@
// then remove the attribute
// if the job has not yet finished, remove the attribute sothat it
// gets refreshed.
+ boolean isJobComplete = false;
+ if (null != job) {
+ String jobStatus = job.get(Keys.JOB_STATUS);
+ isJobComplete = Values.SUCCESS.name() == jobStatus
+ || Values.FAILED.name() == jobStatus
+ || Values.KILLED.name() == jobStatus;
+ }
if (null != job &&
(!jobId.equals(job.get(Keys.JOBID))
- || job.get(Keys.JOB_STATUS) == "")) {
+ || !isJobComplete)) {
// remove jobInfo from session, keep only one job in session at a time
request.getSession().removeAttribute("job");
job = null ;
