Author: sharad
Date: Mon Jun 6 09:05:50 2011
New Revision: 1132556
URL: http://svn.apache.org/viewvc?rev=1132556&view=rev
Log:
Fix class cast exception in Task abort for old mapreduce apis.
Modified:
hadoop/mapreduce/branches/MR-279/CHANGES.txt
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskAttemptContextImpl.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMROldApiJobs.java
Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1132556&r1=1132555&r2=1132556&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Mon Jun 6 09:05:50 2011
@@ -4,6 +4,7 @@ Trunk (unreleased changes)
MAPREDUCE-279
+ Fix class cast exception in Task abort for old mapreduce apis. (sharad)
Fix various issues with Web UI's. (Luke Lu)
Modified:
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1132556&r1=1132555&r2=1132556&view=diff
==============================================================================
---
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
(original)
+++
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
Mon Jun 6 09:05:50 2011
@@ -44,6 +44,7 @@ import org.apache.hadoop.io.DataOutputBu
import org.apache.hadoop.mapred.MapReduceChildJVM;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.mapred.Task;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.WrappedJvmID;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.OutputCommitter;
@@ -57,7 +58,7 @@ import org.apache.hadoop.mapreduce.jobhi
import
org.apache.hadoop.mapreduce.jobhistory.TaskAttemptUnsuccessfulCompletionEvent;
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
-import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.apache.hadoop.mapred.TaskAttemptContextImpl;
import org.apache.hadoop.mapreduce.v2.MRConstants;
import org.apache.hadoop.mapreduce.v2.api.records.Counters;
import org.apache.hadoop.mapreduce.v2.api.records.Phase;
@@ -1015,7 +1016,7 @@ public abstract class TaskAttemptImpl im
public void transition(TaskAttemptImpl taskAttempt,
TaskAttemptEvent event) {
TaskAttemptContext taskContext =
- new TaskAttemptContextImpl(taskAttempt.conf,
+ new TaskAttemptContextImpl(new JobConf(taskAttempt.conf),
TypeConverter.fromYarn(taskAttempt.attemptId));
taskAttempt.eventHandler.handle(new TaskCleanupEvent(
taskAttempt.attemptId,
Modified:
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskAttemptContextImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskAttemptContextImpl.java?rev=1132556&r1=1132555&r2=1132556&view=diff
==============================================================================
---
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskAttemptContextImpl.java
(original)
+++
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskAttemptContextImpl.java
Mon Jun 6 09:05:50 2011
@@ -33,7 +33,7 @@ public class TaskAttemptContextImpl
implements TaskAttemptContext {
private Progressable progress;
- TaskAttemptContextImpl(JobConf conf, TaskAttemptID taskid) {
+ public TaskAttemptContextImpl(JobConf conf, TaskAttemptID taskid) {
this(conf, taskid, Reporter.NULL);
}
Modified:
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java?rev=1132556&r1=1132555&r2=1132556&view=diff
==============================================================================
---
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
(original)
+++
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
Mon Jun 6 09:05:50 2011
@@ -157,7 +157,7 @@ public class TestMRJobs {
// JobStatus?)--compare against MRJobConfig.JOB_UBERTASK_ENABLE value
}
- //@Test
+ @Test
public void testRandomWriter() throws IOException, InterruptedException,
ClassNotFoundException {
@@ -199,7 +199,7 @@ public class TestMRJobs {
// TODO later: add explicit "isUber()" checks of some sort
}
- // @Test
+ @Test
public void testFailingMapper() throws IOException, InterruptedException,
ClassNotFoundException {
Modified:
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMROldApiJobs.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMROldApiJobs.java?rev=1132556&r1=1132555&r2=1132556&view=diff
==============================================================================
---
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMROldApiJobs.java
(original)
+++
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMROldApiJobs.java
Mon Jun 6 09:05:50 2011
@@ -29,6 +29,7 @@ import org.apache.hadoop.FailMapper;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
@@ -40,6 +41,7 @@ import org.apache.hadoop.mapred.TextInpu
import org.apache.hadoop.mapred.lib.IdentityMapper;
import org.apache.hadoop.mapred.lib.IdentityReducer;
import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -50,9 +52,18 @@ public class TestMROldApiJobs {
private static final Log LOG = LogFactory.getLog(TestMROldApiJobs.class);
protected static MiniMRYarnCluster mrCluster;
+ private static Configuration conf = new Configuration();
+ private static FileSystem localFs;
+ static {
+ try {
+ localFs = FileSystem.getLocal(conf);
+ } catch (IOException io) {
+ throw new RuntimeException("problem getting local fs", io);
+ }
+ }
@BeforeClass
- public static void setup() {
+ public static void setup() throws IOException {
if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
@@ -69,6 +80,11 @@ public class TestMROldApiJobs {
// TestMRJobs is for testing non-uberized operation only; see TestUberAM
// for corresponding uberized tests.
mrCluster.getConfig().setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
+
+ // Copy MRAppJar and make it private. TODO: FIXME. This is a hack to
+ // workaround the absent public discache.
+ localFs.copyFromLocalFile(new Path(MiniMRYarnCluster.APPJAR),
TestMRJobs.APP_JAR);
+ localFs.setPermission(TestMRJobs.APP_JAR, new FsPermission("700"));
}
@AfterClass
@@ -142,6 +158,7 @@ public class TestMROldApiJobs {
throws IOException, InterruptedException {
conf.setJobName("test-job-fail");
conf.setMapperClass(FailMapper.class);
+ conf.setJarByClass(FailMapper.class);
conf.setReducerClass(IdentityReducer.class);
conf.setMaxMapAttempts(1);
@@ -154,6 +171,7 @@ public class TestMROldApiJobs {
throws IOException, InterruptedException {
conf.setJobName("test-job-succeed");
conf.setMapperClass(IdentityMapper.class);
+ //conf.setJar(new File(MiniMRYarnCluster.APPJAR).getAbsolutePath());
conf.setReducerClass(IdentityReducer.class);
boolean success = runJob(conf, inDir, outDir, 1 , 1);
@@ -176,9 +194,9 @@ public class TestMROldApiJobs {
DataOutputStream file = fs.create(new Path(inDir, "part-" + i));
file.writeBytes(input);
file.close();
- }
+ }
- conf.setJar(new File(MiniMRYarnCluster.APPJAR).getAbsolutePath());
+ DistributedCache.addFileToClassPath(TestMRJobs.APP_JAR, conf);
conf.setOutputCommitter(CustomOutputCommitter.class);
conf.setInputFormat(TextInputFormat.class);
conf.setOutputKeyClass(LongWritable.class);
@@ -190,6 +208,7 @@ public class TestMROldApiJobs {
conf.setNumReduceTasks(numReds);
JobClient jobClient = new JobClient(conf);
+
RunningJob job = jobClient.submitJob(conf);
return jobClient.monitorAndPrintJob(conf, job);
}
