Author: omalley
Date: Tue Mar 8 06:00:41 2011
New Revision: 1079253
URL: http://svn.apache.org/viewvc?rev=1079253&view=rev
Log:
commit ab07c14c828e506daf7e804eb90aa632a558b575
Author: Greg Roelofs <[email protected]>
Date: Tue Jan 25 15:05:21 2011 -0800
Fixes for two more failing unit tests (same test, diff APIs) and for assert
messages in third. (Last two remaining tests fail in uber and error/crash
in
non-uber.)
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestLazyOutput.java
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestLazyOutput.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestLazyOutput.java?rev=1079253&r1=1079252&r2=1079253&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestLazyOutput.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestLazyOutput.java
Tue Mar 8 06:00:41 2011
@@ -108,6 +108,9 @@ public class TestLazyOutput extends Test
job.setMapperClass(TestMapper.class);
job.setReducerClass(TestReducer.class);
+ // ubertask has just one output file (part-r-00000) != number of maptasks
+ job.setBoolean(JobContext.JOB_UBERTASK_ENABLE, false);
+
JobClient client = new JobClient(job);
job.setNumReduceTasks(numReducers);
if (createLazily) {
@@ -161,7 +164,8 @@ public class TestLazyOutput extends Test
for(int i=0; i < fileList.length; ++i) {
System.out.println("Test1 File list[" + i + "]" + ": "+ fileList[i]);
}
- assertTrue(fileList.length == (numReducers - 1));
+ assertEquals("Test1: wrong number of files.",
+ numReducers - 1, fileList.length);
// Test 2. 0 Reducers, maps directly write to the output files
Path output2 = new Path("/testlazy/output2");
@@ -174,7 +178,8 @@ public class TestLazyOutput extends Test
System.out.println("Test2 File list[" + i + "]" + ": "+ fileList[i]);
}
- assertTrue(fileList.length == numMappers - 1);
+ assertEquals("Test2: wrong number of files.",
+ numMappers - 1, fileList.length);
// Test 3. 0 Reducers, but flag is turned off
Path output3 = new Path("/testlazy/output3");
@@ -187,12 +192,12 @@ public class TestLazyOutput extends Test
System.out.println("Test3 File list[" + i + "]" + ": "+ fileList[i]);
}
- assertTrue(fileList.length == numMappers);
+ assertEquals("Test3: wrong number of files.",
+ numMappers, fileList.length);
} finally {
if (dfs != null) { dfs.shutdown(); }
- if (mr != null) { mr.shutdown();
- }
+ if (mr != null) { mr.shutdown(); }
}
}
}
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java?rev=1079253&r1=1079252&r2=1079253&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java
Tue Mar 8 06:00:41 2011
@@ -244,22 +244,23 @@ public class TestUserLogCleanup {
taskLogCleanupThread = new UserLogCleaner(conf, taskController);
myClock = new FakeClock(); // clock is reset.
taskLogCleanupThread.setClock(myClock);
+ // [FIXME: race condition in here? sometimes works, sometimes fails...]
taskLogCleanupThread.clearOldUserLogs(conf);
tt.setTaskLogCleanupThread(taskLogCleanupThread);
- assertFalse(foo.exists());
- assertFalse(bar.exists());
- assertTrue(jobUserlog1.exists());
- assertTrue(jobUserlog2.exists());
- assertTrue(jobUserlog3.exists());
- assertTrue(jobUserlog4.exists());
+ assertFalse("userlog subdir 'foo' exists.", foo.exists());
+ assertFalse("userlog file 'bar' exists.", bar.exists());
+ assertTrue("userlog for job 1 missing.", jobUserlog1.exists());
+ assertTrue("userlog for job 2 missing.", jobUserlog2.exists());
+ assertTrue("userlog for job 3 missing.", jobUserlog3.exists());
+ assertTrue("userlog for job 4 missing.", jobUserlog4.exists());
myClock.advance(ONE_HOUR);
// time is now 1.
taskLogCleanupThread.processCompletedJobs();
- assertTrue(jobUserlog1.exists());
- assertTrue(jobUserlog2.exists());
- assertTrue(jobUserlog3.exists());
- assertTrue(jobUserlog4.exists());
+ assertTrue("userlog for job 1 missing.", jobUserlog1.exists());
+ assertTrue("userlog for job 2 missing.", jobUserlog2.exists());
+ assertTrue("userlog for job 3 missing.", jobUserlog3.exists());
+ assertTrue("userlog for job 4 missing.", jobUserlog4.exists());
// mimic localizeJob followed KillJobAction for jobid3
// add the job for deletion with retainhours = 3.
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java?rev=1079253&r1=1079252&r2=1079253&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java
Tue Mar 8 06:00:41 2011
@@ -110,7 +110,7 @@ public class TestMapReduceLazyOutput ext
} else {
job.setOutputFormatClass(TextOutputFormat.class);
}
- assertTrue(job.waitForCompletion(true));
+ assertTrue("Job failed.", job.waitForCompletion(true));
}
public void createInput(FileSystem fs, int numMappers) throws Exception {
@@ -154,7 +154,8 @@ public class TestMapReduceLazyOutput ext
for(int i=0; i < fileList.length; ++i) {
System.out.println("Test1 File list[" + i + "]" + ": "+ fileList[i]);
}
- assertTrue(fileList.length == (numReducers - 1));
+ assertEquals("Test1: wrong number of files.",
+ numReducers - 1, fileList.length);
// Test 2. 0 Reducers, maps directly write to the output files
Path output2 = new Path("/testlazy/output2");
@@ -167,7 +168,8 @@ public class TestMapReduceLazyOutput ext
System.out.println("Test2 File list[" + i + "]" + ": "+ fileList[i]);
}
- assertTrue(fileList.length == numMappers - 1);
+ assertEquals("Test2: wrong number of files.",
+ numMappers - 1, fileList.length);
// Test 3. 0 Reducers, but flag is turned off
Path output3 = new Path("/testlazy/output3");
@@ -180,12 +182,12 @@ public class TestMapReduceLazyOutput ext
System.out.println("Test3 File list[" + i + "]" + ": "+ fileList[i]);
}
- assertTrue(fileList.length == numMappers);
+ assertEquals("Test3: wrong number of files.",
+ numMappers, fileList.length);
} finally {
if (dfs != null) { dfs.shutdown(); }
- if (mr != null) { mr.shutdown();
- }
+ if (mr != null) { mr.shutdown(); }
}
}
