Author: ddas
Date: Thu Jan 3 00:15:30 2008
New Revision: 608373
URL: http://svn.apache.org/viewvc?rev=608373&view=rev
Log:
HADOOP-2208. Counter update frequency (from TaskTracker to JobTracker) is
capped at 1 minute. Contributed by Amareshwari Sri Ramadasu.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskStatus.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=608373&r1=608372&r2=608373&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Thu Jan 3 00:15:30 2008
@@ -152,6 +152,9 @@
HADOOP-1900. Makes the heartbeat and task event queries interval
dependent on the cluster size. (Amareshwari Sri Ramadasu via ddas)
+ HADOOP-2208. Counter update frequency (from TaskTracker to JobTracker) is
+ capped at 1 minute. (Amareshwari Sri Ramadasu via ddas)
+
BUG FIXES
HADOOP-2100. Remove faulty check for existence of $HADOOP_PID_DIR and let
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java?rev=608373&r1=608372&r2=608373&view=diff
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java
Thu Jan 3 00:15:30 2008
@@ -39,8 +39,9 @@
* TaskTrackerStatus for HADOOP-1274
* Version 8: HeartbeatResponse is added with the next heartbeat interval.
* version 9 changes the counter representation for HADOOP-2248
+ * version 10 changes the TaskStatus representation for HADOOP-2208
*/
- public static final long versionID = 9L;
+ public static final long versionID = 10L;
public final static int TRACKERS_OK = 0;
public final static int UNKNOWN_TASKTRACKER = 1;
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java?rev=608373&r1=608372&r2=608373&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java Thu
Jan 3 00:15:30 2008
@@ -29,6 +29,8 @@
public static final int CLUSTER_INCREMENT = 50;
+ public static final long COUNTER_UPDATE_INTERVAL = 60 * 1000;
+
//for the inmemory filesystem (to do in-memory merge)
/**
* Constant denoting when a merge of in memory files will be triggered
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskStatus.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskStatus.java?rev=608373&r1=608372&r2=608373&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskStatus.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskStatus.java Thu
Jan 3 00:15:30 2008
@@ -55,6 +55,7 @@
private Phase phase = Phase.STARTING;
private Counters counters;
+ private boolean includeCounters;
public TaskStatus() {}
@@ -70,6 +71,7 @@
this.taskTracker = taskTracker;
this.phase = phase;
this.counters = counters;
+ this.includeCounters = true;
}
public String getTaskId() { return taskid; }
@@ -181,6 +183,15 @@
}
this.phase = phase;
}
+
+ public boolean getIncludeCounters() {
+ return includeCounters;
+ }
+
+ public void setIncludeCounters(boolean send) {
+ includeCounters = send;
+ }
+
/**
* Get task's counters.
*/
@@ -281,7 +292,10 @@
WritableUtils.writeEnum(out, phase);
out.writeLong(startTime);
out.writeLong(finishTime);
- counters.write(out);
+ out.writeBoolean(includeCounters);
+ if (includeCounters) {
+ counters.write(out);
+ }
}
public void readFields(DataInput in) throws IOException {
@@ -294,7 +308,10 @@
this.startTime = in.readLong();
this.finishTime = in.readLong();
counters = new Counters();
- counters.readFields(in);
+ this.includeCounters = in.readBoolean();
+ if (includeCounters) {
+ counters.readFields(in);
+ }
}
//////////////////////////////////////////////////////////////////////////////
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=608373&r1=608372&r2=608373&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Thu
Jan 3 00:15:30 2008
@@ -958,12 +958,25 @@
return State.NORMAL;
}
+ private long previousUpdate = 0;
+
/**
* Build and transmit the heart beat to the JobTracker
* @return false if the tracker was unknown
* @throws IOException
*/
private HeartbeatResponse transmitHeartBeat() throws IOException {
+ // Send Counters in the status once every COUNTER_UPDATE_INTERVAL
+ long now = System.currentTimeMillis();
+ boolean sendCounters;
+ if (now > (previousUpdate + COUNTER_UPDATE_INTERVAL)) {
+ sendCounters = true;
+ previousUpdate = now;
+ }
+ else {
+ sendCounters = false;
+ }
+
//
// Check if the last heartbeat got through...
// if so then build the heartbeat information for the JobTracker;
@@ -972,7 +985,9 @@
if (status == null) {
synchronized (this) {
status = new TaskTrackerStatus(taskTrackerName, localHostname,
- httpPort,
cloneAndResetRunningTaskStatuses(),
+ httpPort,
+ cloneAndResetRunningTaskStatuses(
+ sendCounters),
failures,
maxCurrentMapTasks,
maxCurrentReduceTasks);
@@ -2060,10 +2075,16 @@
return taskTrackerName;
}
- private synchronized List<TaskStatus> cloneAndResetRunningTaskStatuses() {
+ private synchronized List<TaskStatus> cloneAndResetRunningTaskStatuses(
+ boolean sendCounters) {
List<TaskStatus> result = new ArrayList<TaskStatus>(runningTasks.size());
for(TaskInProgress tip: runningTasks.values()) {
TaskStatus status = tip.getStatus();
+ status.setIncludeCounters(sendCounters);
+ // send counters for finished or failed tasks.
+ if (status.getRunState() != TaskStatus.State.RUNNING) {
+ status.setIncludeCounters(true);
+ }
result.add((TaskStatus)status.clone());
status.clearStatus();
}
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java?rev=608373&r1=608372&r2=608373&view=diff
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java
Thu Jan 3 00:15:30 2008
@@ -37,8 +37,9 @@
* org.apache.hadoop.mapred.TaskStatus.Phase, Counters)
* with [EMAIL PROTECTED] #statusUpdate(String, TaskStatus)}
* Version 5 changed counters representation for HADOOP-2248
+ * Version 6 changes the TaskStatus representation for HADOOP-2208
* */
- public static final long versionID = 5L;
+ public static final long versionID = 6L;
/** Called when a child task process starts, to get its task.*/
Task getTask(String taskid) throws IOException;
