Author: ddas
Date: Thu Jan 22 13:31:19 2009
New Revision: 736802
URL: http://svn.apache.org/viewvc?rev=736802&view=rev
Log:
Merge -r 736800:736801 from trunk onto 0.20 branch. Fixes HADOOP-4939.
Added:
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/ReliabilityTest.java
- copied unchanged from r736801,
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/ReliabilityTest.java
Modified:
hadoop/core/branches/branch-0.20/CHANGES.txt
hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/util/Shell.java
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobClient.java
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobSubmissionProtocol.java
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/test/AllTestDriver.java
Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Thu Jan 22 13:31:19 2009
@@ -311,6 +311,9 @@
HADOOP-4828. Updates documents to do with configuration (HADOOP-4631).
(Sharad Agarwal via ddas)
+ HADOOP-4939. Adds a test that would inject random failures for tasks in
+ large jobs and would also inject TaskTracker failures. (ddas)
+
OPTIMIZATIONS
HADOOP-3293. Fixes FileInputFormat to do provide locations for splits
Modified:
hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/util/Shell.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/util/Shell.java?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/util/Shell.java
(original)
+++ hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/util/Shell.java
Thu Jan 22 13:31:19 2009
@@ -334,7 +334,23 @@
* @return the output of the executed command.
*/
public static String execCommand(String ... cmd) throws IOException {
+ return execCommand(null, cmd);
+ }
+
+ /**
+ * Static method to execute a shell command.
+ * Covers most of the simple cases without requiring the user to implement
+ * the <code>Shell</code> interface.
+ * @param env the map of environment key=value
+ * @param cmd shell command to execute.
+ * @return the output of the executed command.
+ */
+ public static String execCommand(Map<String,String> env, String ... cmd)
+ throws IOException {
ShellCommandExecutor exec = new ShellCommandExecutor(cmd);
+ if (env != null) {
+ exec.setEnvironment(env);
+ }
exec.execute();
return exec.getOutput();
}
Modified:
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java
(original)
+++
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java
Thu Jan 22 13:31:19 2009
@@ -61,6 +61,7 @@
private Collection<String> activeTrackers = new ArrayList<String>();
private Collection<String> blacklistedTrackers = new ArrayList<String>();
private int numBlacklistedTrackers;
+ private long ttExpiryInterval;
private int map_tasks;
private int reduce_tasks;
private int max_map_tasks;
@@ -85,7 +86,8 @@
@Deprecated
ClusterStatus(int trackers, int maps, int reduces, int maxMaps,
int maxReduces, JobTracker.State state) {
- this(trackers, 0, maps, reduces, maxMaps, maxReduces, state);
+ this(trackers, 0, JobTracker.TASKTRACKER_EXPIRY_INTERVAL, maps, reduces,
+ maxMaps, maxReduces, state);
}
/**
@@ -93,16 +95,19 @@
*
* @param trackers no. of tasktrackers in the cluster
* @param blacklists no of blacklisted task trackers in the cluster
+ * @param ttExpiryInterval the tasktracker expiry interval
* @param maps no. of currently running map-tasks in the cluster
* @param reduces no. of currently running reduce-tasks in the cluster
* @param maxMaps the maximum no. of map tasks in the cluster
* @param maxReduces the maximum no. of reduce tasks in the cluster
* @param state the {...@link JobTracker.State} of the
<code>JobTracker</code>
*/
- ClusterStatus(int trackers, int blacklists, int maps, int reduces,
+ ClusterStatus(int trackers, int blacklists, long ttExpiryInterval,
+ int maps, int reduces,
int maxMaps, int maxReduces, JobTracker.State state) {
numActiveTrackers = trackers;
numBlacklistedTrackers = blacklists;
+ this.ttExpiryInterval = ttExpiryInterval;
map_tasks = maps;
reduce_tasks = reduces;
max_map_tasks = maxMaps;
@@ -117,6 +122,7 @@
*
* @param activeTrackers active tasktrackers in the cluster
* @param blacklistedTrackers blacklisted tasktrackers in the cluster
+ * @param ttExpiryInterval the tasktracker expiry interval
* @param maps no. of currently running map-tasks in the cluster
* @param reduces no. of currently running reduce-tasks in the cluster
* @param maxMaps the maximum no. of map tasks in the cluster
@@ -125,10 +131,11 @@
*/
ClusterStatus(Collection<String> activeTrackers,
Collection<String> blacklistedTrackers,
+ long ttExpiryInterval,
int maps, int reduces, int maxMaps, int maxReduces,
JobTracker.State state) {
- this(activeTrackers.size(), blacklistedTrackers.size(), maps, reduces,
- maxMaps, maxReduces, state);
+ this(activeTrackers.size(), blacklistedTrackers.size(), ttExpiryInterval,
+ maps, reduces, maxMaps, maxReduces, state);
this.activeTrackers = activeTrackers;
this.blacklistedTrackers = blacklistedTrackers;
}
@@ -171,6 +178,14 @@
}
/**
+ * Get the tasktracker expiry interval for the cluster
+ * @return the expiry interval in msec
+ */
+ public long getTTExpiryInterval() {
+ return ttExpiryInterval;
+ }
+
+ /**
* Get the number of currently running map tasks in the cluster.
*
* @return the number of currently running map tasks in the cluster.
@@ -255,6 +270,7 @@
Text.writeString(out, tracker);
}
}
+ out.writeLong(ttExpiryInterval);
out.writeInt(map_tasks);
out.writeInt(reduce_tasks);
out.writeInt(max_map_tasks);
@@ -281,6 +297,7 @@
blacklistedTrackers.add(name);
}
}
+ ttExpiryInterval = in.readLong();
map_tasks = in.readInt();
reduce_tasks = in.readInt();
max_map_tasks = in.readInt();
Modified:
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobClient.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobClient.java?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobClient.java
(original)
+++
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobClient.java
Thu Jan 22 13:31:19 2009
@@ -1193,7 +1193,20 @@
* @throws IOException
*/
public ClusterStatus getClusterStatus() throws IOException {
- return jobSubmitClient.getClusterStatus(false);
+ return getClusterStatus(false);
+ }
+
+ /**
+ * Get status information about the Map-Reduce cluster.
+ *
+ * @param detailed if true then get a detailed status including the
+ * tracker names
+ * @return the status information about the Map-Reduce cluster as an object
+ * of {...@link ClusterStatus}.
+ * @throws IOException
+ */
+ public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
+ return jobSubmitClient.getClusterStatus(detailed);
}
Modified:
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobSubmissionProtocol.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobSubmissionProtocol.java?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobSubmissionProtocol.java
(original)
+++
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobSubmissionProtocol.java
Thu Jan 22 13:31:19 2009
@@ -55,9 +55,11 @@
* for HADOOP-4305
* Version 19: Modified TaskReport to have TIP status and modified the
* method getClusterStatus() to take a boolean argument
- * for HADOOP-4807
+ * for HADOOP-4807
+ * Version 20: Modified ClusterStatus to have the tasktracker expiry
+ * interval for HADOOP-4939
*/
- public static final long versionID = 19L;
+ public static final long versionID = 20L;
/**
* Allocate a name for the job.
Modified:
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
(original)
+++
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
Thu Jan 22 13:31:19 2009
@@ -2711,6 +2711,7 @@
List<List<String>> trackerNames = taskTrackerNames();
return new ClusterStatus(trackerNames.get(0),
trackerNames.get(1),
+ TASKTRACKER_EXPIRY_INTERVAL,
totalMaps,
totalReduces,
totalMapTaskCapacity,
@@ -2720,6 +2721,7 @@
return new ClusterStatus(taskTrackers.size() -
getBlacklistedTrackerCount(),
getBlacklistedTrackerCount(),
+ TASKTRACKER_EXPIRY_INTERVAL,
totalMaps,
totalReduces,
totalMapTaskCapacity,
Modified:
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
(original)
+++
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
Thu Jan 22 13:31:19 2009
@@ -386,7 +386,7 @@
}
public ClusterStatus getClusterStatus(boolean detailed) {
- return new ClusterStatus(1, 0, map_tasks, reduce_tasks, 1, 1,
+ return new ClusterStatus(1, 0, 0, map_tasks, reduce_tasks, 1, 1,
JobTracker.State.RUNNING);
}
Modified:
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
(original)
+++
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
Thu Jan 22 13:31:19 2009
@@ -135,7 +135,9 @@
@Override
public ClusterStatus getClusterStatus() {
int numTrackers = trackers.size();
- return new ClusterStatus(numTrackers, 0, maps, reduces,
+ return new ClusterStatus(numTrackers, 0,
+ JobTracker.TASKTRACKER_EXPIRY_INTERVAL,
+ maps, reduces,
numTrackers * maxMapTasksPerTracker,
numTrackers * maxReduceTasksPerTracker,
JobTracker.State.RUNNING);
Modified:
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/test/AllTestDriver.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/test/AllTestDriver.java?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/test/AllTestDriver.java
(original)
+++
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/test/AllTestDriver.java
Thu Jan 22 13:31:19 2009
@@ -22,6 +22,7 @@
import org.apache.hadoop.mapred.BigMapOutput;
import org.apache.hadoop.mapred.GenericMRLoadGenerator;
import org.apache.hadoop.mapred.MRBench;
+import org.apache.hadoop.mapred.ReliabilityTest;
import org.apache.hadoop.mapred.SortValidator;
import org.apache.hadoop.mapred.TestMapRed;
import org.apache.hadoop.mapred.TestSequenceFileInputFormat;
@@ -74,6 +75,9 @@
pgd.addClass("filebench", FileBench.class, "Benchmark
SequenceFile(Input|Output)Format (block,record compressed and uncompressed),
Text(Input|Output)Format (compressed and uncompressed)");
pgd.addClass("dfsthroughput", BenchmarkThroughput.class,
"measure hdfs throughput");
+ pgd.addClass("MRReliabilityTest", ReliabilityTest.class,
+ "A program that tests the reliability of the MR framework by " +
+ "injecting faults/failures");
pgd.driver(argv);
} catch(Throwable e) {
e.printStackTrace();
