Author: todd
Date: Fri Dec 31 18:14:36 2010
New Revision: 1054154
URL: http://svn.apache.org/viewvc?rev=1054154&view=rev
Log:
MAPREDUCE-1906. Lower minimum heartbeat interval for TaskTracker. Contributed
by Scott Carey and Todd Lipcon
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java
hadoop/mapreduce/trunk/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/HeartbeatResponse.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapredHeartbeat.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestParallelInitialization.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1054154&r1=1054153&r2=1054154&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Dec 31 18:14:36 2010
@@ -18,6 +18,9 @@ Trunk (unreleased changes)
MAPREDUCE-1831. BlockPlacement policy for HDFS-RAID.
(Scott Chen via dhruba)
+ MAPREDUCE-1906. Lower minimum heartbeat interval for TaskTracker
+ (Scott Carey and Todd Lipcon via todd)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java?rev=1054154&r1=1054153&r2=1054154&view=diff
==============================================================================
---
hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java
(original)
+++
hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java
Fri Dec 31 18:14:36 2010
@@ -42,6 +42,7 @@ import org.apache.hadoop.mapreduce.Queue
import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName;
import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobHistory;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
import org.apache.hadoop.mapreduce.split.JobSplit;
import org.apache.hadoop.security.authorize.AccessControlList;
@@ -677,7 +678,7 @@ public class CapacityTestUtils {
}
public int getNextHeartbeatInterval() {
- return MRConstants.HEARTBEAT_INTERVAL_MIN;
+ return JTConfig.JT_HEARTBEAT_INTERVAL_MIN_DEFAULT;
}
/**
Modified:
hadoop/mapreduce/trunk/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java?rev=1054154&r1=1054153&r2=1054154&view=diff
==============================================================================
---
hadoop/mapreduce/trunk/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
(original)
+++
hadoop/mapreduce/trunk/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
Fri Dec 31 18:14:36 2010
@@ -43,6 +43,7 @@ import org.apache.hadoop.mapred.FakeObje
import org.apache.hadoop.mapred.JobInProgress.KillInterruptedException;
import org.apache.hadoop.mapred.UtilsForTests.FakeClock;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
import org.apache.hadoop.mapreduce.split.JobSplit;
import org.apache.hadoop.net.Node;
@@ -406,7 +407,7 @@ public class TestFairScheduler extends T
@Override
public int getNextHeartbeatInterval() {
- return MRConstants.HEARTBEAT_INTERVAL_MIN;
+ return JTConfig.JT_HEARTBEAT_INTERVAL_MIN_DEFAULT;
}
@Override
Modified:
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/HeartbeatResponse.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/HeartbeatResponse.java?rev=1054154&r1=1054153&r2=1054154&view=diff
==============================================================================
---
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/HeartbeatResponse.java
(original)
+++
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/HeartbeatResponse.java
Fri Dec 31 18:14:36 2010
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configurab
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
/**
* The response sent by the {...@link JobTracker} to the hearbeat sent
@@ -43,7 +44,7 @@ class HeartbeatResponse implements Writa
HeartbeatResponse(short responseId, TaskTrackerAction[] actions) {
this.responseId = responseId;
this.actions = actions;
- this.heartbeatInterval = MRConstants.HEARTBEAT_INTERVAL_MIN;
+ this.heartbeatInterval = JTConfig.JT_HEARTBEAT_INTERVAL_MIN_DEFAULT;
}
public void setResponseId(short responseId) {
Modified:
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=1054154&r1=1054153&r2=1054154&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
(original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
Fri Dec 31 18:14:36 2010
@@ -159,6 +159,9 @@ public class JobTracker implements MRCon
private float HEARTBEATS_SCALING_FACTOR;
private final float MIN_HEARTBEATS_SCALING_FACTOR = 0.01f;
private final float DEFAULT_HEARTBEATS_SCALING_FACTOR = 1.0f;
+
+ // Minimum interval for heartbeats regardless of cluster size.
+ private int HEARTBEAT_INTERVAL_MIN;
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -1422,6 +1425,9 @@ public class JobTracker implements MRCon
HEARTBEATS_SCALING_FACTOR = DEFAULT_HEARTBEATS_SCALING_FACTOR;
}
+ HEARTBEAT_INTERVAL_MIN = conf.getInt(JT_HEARTBEAT_INTERVAL_MIN,
+ JT_HEARTBEAT_INTERVAL_MIN_DEFAULT);
+
//This configuration is there solely for tuning purposes and
//once this feature has been tested in real clusters and an appropriate
//value for the threshold has been found, this config might be taken out.
@@ -2520,7 +2526,7 @@ public class JobTracker implements MRCon
int clusterSize = getClusterStatus().getTaskTrackers();
int heartbeatInterval = Math.max(
(int)(1000 * HEARTBEATS_SCALING_FACTOR *
- Math.ceil((double)clusterSize /
+ ((double)clusterSize /
NUM_HEARTBEATS_IN_SECOND)),
HEARTBEAT_INTERVAL_MIN) ;
return heartbeatInterval;
Modified:
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java?rev=1054154&r1=1054153&r2=1054154&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java
(original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java
Fri Dec 31 18:14:36 2010
@@ -25,8 +25,6 @@ interface MRConstants {
//
// Timeouts, constants
//
- public static final int HEARTBEAT_INTERVAL_MIN = 3 * 1000;
-
public static final long COUNTER_UPDATE_INTERVAL = 60 * 1000;
//
Modified:
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=1054154&r1=1054153&r2=1054154&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
(original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
Fri Dec 31 18:14:36 2010
@@ -88,6 +88,7 @@ import org.apache.hadoop.mapreduce.secur
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
+import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
import org.apache.hadoop.mapreduce.server.tasktracker.Localizer;
import org.apache.hadoop.mapreduce.task.reduce.ShuffleHeader;
@@ -283,7 +284,8 @@ public class TaskTracker
/**
* the minimum interval between jobtracker polls
*/
- private volatile int heartbeatInterval = HEARTBEAT_INTERVAL_MIN;
+ private volatile int heartbeatInterval =
+ JTConfig.JT_HEARTBEAT_INTERVAL_MIN_DEFAULT;
/**
* Number of maptask completion events locations to poll for at one time
*/
Modified:
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java?rev=1054154&r1=1054153&r2=1054154&view=diff
==============================================================================
---
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java
(original)
+++
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java
Fri Dec 31 18:14:36 2010
@@ -47,6 +47,9 @@ public interface JTConfig extends MRConf
"mapreduce.jobtracker.heartbeats.in.second";
public static final String JT_HEARTBEATS_SCALING_FACTOR =
"mapreduce.jobtracker.heartbeats.scaling.factor";
+ public static final String JT_HEARTBEAT_INTERVAL_MIN =
+ "mapreduce.jobtracker.heartbeat.interval.min";
+ public static final int JT_HEARTBEAT_INTERVAL_MIN_DEFAULT = 300;
public static final String JT_PERSIST_JOBSTATUS =
"mapreduce.jobtracker.persist.jobstatus.active";
public static final String JT_PERSIST_JOBSTATUS_HOURS =
Modified:
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java?rev=1054154&r1=1054153&r2=1054154&view=diff
==============================================================================
---
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
(original)
+++
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
Fri Dec 31 18:14:36 2010
@@ -29,6 +29,7 @@ import junit.framework.TestCase;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.mapred.JobStatusChangeEvent.EventType;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
import org.apache.hadoop.mapreduce.split.JobSplit;
@@ -190,7 +191,7 @@ public class TestJobQueueTaskScheduler e
@Override
public int getNextHeartbeatInterval() {
- return MRConstants.HEARTBEAT_INTERVAL_MIN;
+ return JTConfig.JT_HEARTBEAT_INTERVAL_MIN_DEFAULT;
}
@Override
Modified:
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapredHeartbeat.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapredHeartbeat.java?rev=1054154&r1=1054153&r2=1054154&view=diff
==============================================================================
---
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapredHeartbeat.java
(original)
+++
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapredHeartbeat.java
Fri Dec 31 18:14:36 2010
@@ -43,7 +43,7 @@ public class TestMapredHeartbeat extends
while(jc.getClusterStatus().getTaskTrackers() != taskTrackers) {
UtilsForTests.waitFor(100);
}
- assertEquals(MRConstants.HEARTBEAT_INTERVAL_MIN,
+ assertEquals(JTConfig.JT_HEARTBEAT_INTERVAL_MIN_DEFAULT,
mr.getJobTrackerRunner().getJobTracker().getNextHeartbeatInterval());
mr.shutdown();
@@ -62,14 +62,14 @@ public class TestMapredHeartbeat extends
// test configured heartbeat interval is capped with min value
taskTrackers = 5;
- conf.setInt(JTConfig.JT_HEARTBEATS_IN_SECOND, 10);
+ conf.setInt(JTConfig.JT_HEARTBEATS_IN_SECOND, 100);
mr = new MiniMRCluster(taskTrackers, "file:///", 3,
null, null, conf);
jc = new JobClient(mr.createJobConf());
while(jc.getClusterStatus().getTaskTrackers() != taskTrackers) {
UtilsForTests.waitFor(100);
}
- assertEquals(MRConstants.HEARTBEAT_INTERVAL_MIN,
+ assertEquals(JTConfig.JT_HEARTBEAT_INTERVAL_MIN_DEFAULT,
mr.getJobTrackerRunner().getJobTracker().getNextHeartbeatInterval());
} finally {
if (mr != null) { mr.shutdown(); }
Modified:
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestParallelInitialization.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestParallelInitialization.java?rev=1054154&r1=1054153&r2=1054154&view=diff
==============================================================================
---
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestParallelInitialization.java
(original)
+++
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestParallelInitialization.java
Fri Dec 31 18:14:36 2010
@@ -30,6 +30,7 @@ import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobHistory;
import org.apache.hadoop.mapred.JobInProgress.KillInterruptedException;
import org.apache.hadoop.mapred.JobStatusChangeEvent.EventType;
+import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
public class TestParallelInitialization extends TestCase {
@@ -131,7 +132,7 @@ public class TestParallelInitialization
}
public int getNextHeartbeatInterval() {
- return MRConstants.HEARTBEAT_INTERVAL_MIN;
+ return JTConfig.JT_HEARTBEAT_INTERVAL_MIN_DEFAULT;
}
public void killJob(JobID jobid) {
