Author: omalley
Date: Mon Apr 6 06:31:11 2009
New Revision: 762218
URL: http://svn.apache.org/viewvc?rev=762218&view=rev
Log:
HADOOP-5437. Fix TestMiniMRDFSSort to properly test jvm-reuse. (omalley)
Modified:
hadoop/core/branches/branch-0.20/ (props changed)
hadoop/core/branches/branch-0.20/CHANGES.txt (contents, props changed)
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
Propchange: hadoop/core/branches/branch-0.20/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Apr 6 06:31:11 2009
@@ -1,2 +1,2 @@
/hadoop/core/branches/branch-0.19:713112
-/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,759398,759932,760502,760783,761046,761482,761632
+/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,759398,759932,760502,760783,761046,761482,761632,762216
Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=762218&r1=762217&r2=762218&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Mon Apr 6 06:31:11 2009
@@ -356,6 +356,8 @@
HADOOP-5468. Add sub-menus to forrest documentation and make some minor
edits. (Corinne Chandel via szetszwo)
+ HADOOP-5437. Fix TestMiniMRDFSSort to properly test jvm-reuse. (omalley)
+
OPTIMIZATIONS
HADOOP-3293. Fixes FileInputFormat to do provide locations for splits
Propchange: hadoop/core/branches/branch-0.20/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Apr 6 06:31:11 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
-/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752514,752555,752590,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755986,755998,756352,757448,757624,757849,758156,759398,759932,760502,760783,761046,761482,761632
+/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752514,752555,752590,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755986,755998,756352,757448,757624,757849,758156,759398,759932,760502,760783,761046,761482,761632,762216
Modified:
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
URL:
http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java?rev=762218&r1=762217&r2=762218&view=diff
==============================================================================
---
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
(original)
+++
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
Mon Apr 6 06:31:11 2009
@@ -18,10 +18,18 @@
package org.apache.hadoop.mapred;
+import java.io.IOException;
+
+import junit.extensions.TestSetup;
+import junit.framework.Test;
import junit.framework.TestCase;
+import junit.framework.TestSuite;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.lib.NullOutputFormat;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.ToolRunner;
@@ -39,9 +47,30 @@
// Knobs to control randomwriter; and hence sort
private static final int NUM_HADOOP_SLAVES = 3;
- private static final int RW_BYTES_PER_MAP = 2 * 1024 * 1024;
+ // make it big enough to cause a spill in the map
+ private static final int RW_BYTES_PER_MAP = 3 * 1024 * 1024;
private static final int RW_MAPS_PER_HOST = 2;
+ private static MiniMRCluster mrCluster = null;
+ private static MiniDFSCluster dfsCluster = null;
+ private static FileSystem dfs = null;
+ public static Test suite() {
+ TestSetup setup = new TestSetup(new TestSuite(TestMiniMRDFSSort.class)) {
+ protected void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ dfsCluster = new MiniDFSCluster(conf, NUM_HADOOP_SLAVES, true, null);
+ dfs = dfsCluster.getFileSystem();
+ mrCluster = new MiniMRCluster(NUM_HADOOP_SLAVES,
+ dfs.getUri().toString(), 1);
+ }
+ protected void tearDown() throws Exception {
+ if (dfsCluster != null) { dfsCluster.shutdown(); }
+ if (mrCluster != null) { mrCluster.shutdown(); }
+ }
+ };
+ return setup;
+ }
+
private static void runRandomWriter(JobConf job, Path sortInput)
throws Exception {
// Scale down the default settings for RandomWriter for the test-case
@@ -57,8 +86,10 @@
private static void runSort(JobConf job, Path sortInput, Path sortOutput)
throws Exception {
+ job.setInt("mapred.job.reuse.jvm.num.tasks", -1);
job.setInt("io.sort.mb", 1);
- job.setLong("mapred.min.split.size", Long.MAX_VALUE);
+ job.setNumMapTasks(12);
+
// Setup command-line arguments to 'sort'
String[] sortArgs = {sortInput.toString(), sortOutput.toString()};
@@ -75,34 +106,66 @@
// Run Sort-Validator
assertEquals(ToolRunner.run(job, new SortValidator(), svArgs), 0);
}
- Configuration conf = new Configuration();
- public void testMapReduceSort() throws Exception {
- MiniDFSCluster dfs = null;
- MiniMRCluster mr = null;
- FileSystem fileSys = null;
- try {
-
- // Start the mini-MR and mini-DFS clusters
- dfs = new MiniDFSCluster(conf, NUM_HADOOP_SLAVES, true, null);
- fileSys = dfs.getFileSystem();
- mr = new MiniMRCluster(NUM_HADOOP_SLAVES, fileSys.getUri().toString(),
1);
-
- // Run randomwriter to generate input for 'sort'
- runRandomWriter(mr.createJobConf(), SORT_INPUT_PATH);
-
- // Run sort
- runSort(mr.createJobConf(), SORT_INPUT_PATH, SORT_OUTPUT_PATH);
-
- // Run sort-validator to check if sort worked correctly
- runSortValidator(mr.createJobConf(), SORT_INPUT_PATH, SORT_OUTPUT_PATH);
- } finally {
- if (dfs != null) { dfs.shutdown(); }
- if (mr != null) { mr.shutdown();
- }
+
+ private static class ReuseDetector extends MapReduceBase
+ implements Mapper<BytesWritable,BytesWritable, Text, Text> {
+ static int instances = 0;
+ Reporter reporter = null;
+
+ @Override
+ public void map(BytesWritable key, BytesWritable value,
+ OutputCollector<Text, Text> output,
+ Reporter reporter) throws IOException {
+ this.reporter = reporter;
+ }
+
+ public void close() throws IOException {
+ reporter.incrCounter("jvm", "use", ++instances);
}
}
- public void testMapReduceSortWithJvmReuse() throws Exception {
- conf.setInt("mapred.job.reuse.jvm.num.tasks", -1);
- testMapReduceSort();
+
+ private static void runJvmReuseTest(JobConf job,
+ boolean reuse) throws IOException {
+ // setup a map-only job that reads the input and only sets the counters
+ // based on how many times the jvm was reused.
+ job.setInt("mapred.job.reuse.jvm.num.tasks", reuse ? -1 : 1);
+ FileInputFormat.setInputPaths(job, SORT_INPUT_PATH);
+ job.setInputFormat(SequenceFileInputFormat.class);
+ job.setOutputFormat(NullOutputFormat.class);
+ job.setMapperClass(ReuseDetector.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(Text.class);
+ job.setNumMapTasks(24);
+ job.setNumReduceTasks(0);
+ RunningJob result = JobClient.runJob(job);
+ long uses = result.getCounters().findCounter("jvm", "use").getValue();
+ System.out.println("maps = " + job.getNumMapTasks());
+ System.out.println(result.getCounters());
+ int maps = job.getNumMapTasks();
+ if (reuse) {
+ assertTrue("maps = " + maps + ", uses = " + uses, maps < uses);
+ } else {
+ assertEquals("uses should be number of maps", job.getNumMapTasks(),
uses);
+ }
+ }
+
+ public void testMapReduceSort() throws Exception {
+ // Run randomwriter to generate input for 'sort'
+ runRandomWriter(mrCluster.createJobConf(), SORT_INPUT_PATH);
+
+ // Run sort
+ runSort(mrCluster.createJobConf(), SORT_INPUT_PATH, SORT_OUTPUT_PATH);
+
+ // Run sort-validator to check if sort worked correctly
+ runSortValidator(mrCluster.createJobConf(), SORT_INPUT_PATH,
+ SORT_OUTPUT_PATH);
+ }
+
+ public void testJvmReuse() throws Exception {
+ runJvmReuseTest(mrCluster.createJobConf(), true);
+ }
+
+ public void testNoJvmReuse() throws Exception {
+ runJvmReuseTest(mrCluster.createJobConf(), false);
}
}
