Author: cutting Date: Tue May 2 16:07:32 2006 New Revision: 399077 URL: http://svn.apache.org/viewcvs?rev=399077&view=rev Log: HADOOP-184. Restructure some test code to better support testing on a cluster. Contributed by Mahadev.
Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ProgramDriver.java lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMapRed.java - copied, changed from r399074, lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MapredLoadTest.java lucene/hadoop/trunk/src/test/org/apache/hadoop/test/ lucene/hadoop/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java Removed: lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MapredLoadTest.java Modified: lucene/hadoop/trunk/CHANGES.txt lucene/hadoop/trunk/build.xml lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/ExampleDriver.java lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java Modified: lucene/hadoop/trunk/CHANGES.txt URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/CHANGES.txt?rev=399077&r1=399076&r2=399077&view=diff ============================================================================== --- lucene/hadoop/trunk/CHANGES.txt (original) +++ lucene/hadoop/trunk/CHANGES.txt Tue May 2 16:07:32 2006 @@ -158,6 +158,9 @@ 41. HADOOP-190. If a child process hangs after it has reported completion, its output should not be lost. (Stack via cutting) +42. HADOOP-184. Re-structure some test code to better support testing + on a cluster. (Mahadev Konar via cutting) + Release 0.1.1 - 2006-04-08 Modified: lucene/hadoop/trunk/build.xml URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/build.xml?rev=399077&r1=399076&r2=399077&view=diff ============================================================================== --- lucene/hadoop/trunk/build.xml (original) +++ lucene/hadoop/trunk/build.xml Tue May 2 16:07:32 2006 @@ -182,7 +182,7 @@ <!-- ================================================================== --> <!-- Compile test code --> <!-- ================================================================== --> - <target name="compile-test" depends="compile"> + <target name="compile-test" depends="compile, jar"> <javac encoding="${build.encoding}" srcdir="${test.src.dir}" @@ -195,6 +195,13 @@ deprecation="${javac.deprecation}"> <classpath refid="test.classpath"/> </javac> + <jar jarfile="${build.dir}/${final.name}-test.jar" + basedir="${test.build.classes}"> + <manifest> + <attribute name="Main-Class" + value="org/apache/hadoop/test/AllTestDriver"/> + </manifest> + </jar> </target> <!-- ================================================================== --> @@ -290,6 +297,8 @@ <copy file="${build.dir}/${final.name}.jar" todir="${dist.dir}"/> <copy file="${build.dir}/${final.name}-examples.jar" todir="${dist.dir}"/> + + <copy file="${build.dir}/${final.name}-test.jar" todir="${dist.dir}"/> <copy todir="${dist.dir}/bin"> <fileset dir="bin"/> Modified: lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/ExampleDriver.java URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/ExampleDriver.java?rev=399077&r1=399076&r2=399077&view=diff ============================================================================== --- lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/ExampleDriver.java (original) +++ lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/ExampleDriver.java Tue May 2 16:07:32 2006 @@ -15,7 +15,7 @@ */ package org.apache.hadoop.examples; - +import org.apache.hadoop.util.ProgramDriver; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.TreeMap; @@ -29,107 +29,21 @@ * A description of an example program based on its class and a * human-readable description. * @author Owen O'Malley - * @date feb 2006 + * @date april 2006 */ - static private class ProgramDescription { - - static final Class[] paramTypes = new Class[] {String[].class}; - - /** - * Create a description of an example program. - * @param mainClass the class with the main for the example program - * @param description a string to display to the user in help messages - * @throws SecurityException if we can't use reflection - * @throws NoSuchMethodException if the class doesn't have a main method - */ - public ProgramDescription(Class mainClass, - String description) - throws SecurityException, NoSuchMethodException { - this.main = mainClass.getMethod("main", paramTypes); - this.description = description; - } - - /** - * Invoke the example application with the given arguments - * @param args the arguments for the application - * @throws Throwable The exception thrown by the invoked method - */ - public void invoke(String[] args) - throws Throwable { - try { - main.invoke(null, new Object[]{args}); - } catch (InvocationTargetException except) { - throw except.getCause(); - } - } - public String getDescription() { - return description; + public static void main(String argv[]){ + ProgramDriver pgd = new ProgramDriver(); + try { + pgd.addClass("wordcount", WordCount.class, + "A map/reduce program that counts the words in the input files."); + pgd.addClass("grep", Grep.class, + "A map/reduce program that counts the matches of a regex in the input."); + pgd.driver(argv); + } + catch(Throwable e){ + e.printStackTrace(); + } } - - private Method main; - private String description; - } - - private static void printUsage(Map programs) { - System.out.println("Valid program names are:"); - for(Iterator itr=programs.entrySet().iterator(); itr.hasNext();) { - Map.Entry item = (Entry) itr.next(); - System.out.println(" " + (String) item.getKey() + ": " + - ((ProgramDescription) item.getValue()).getDescription()); - } - } - - /** - * This is a driver for the example programs. - * It looks at the first command line argument and tries to find an - * example program with that name. - * If it is found, it calls the main method in that class with the rest - * of the command line arguments. - * @param args The argument from the user. args[0] is the command to run. - * @throws NoSuchMethodException - * @throws SecurityException - * @throws IllegalAccessException - * @throws IllegalArgumentException - * @throws Throwable Anything thrown by the example program's main - */ - public static void main(String[] args) - throws Throwable - { - Map programs = new TreeMap(); - - // Add new programs to this list - programs.put("wordcount", new ProgramDescription(WordCount.class, - "A map/reduce program that counts the words in the input files.")); - programs.put("grep", new ProgramDescription(Grep.class, - "A map/reduce program that counts the matches of a regex in the input.")); - programs.put("sort", new ProgramDescription(Sort.class, - "Sort binary keys and values.")); - programs.put("writer", new ProgramDescription(RandomWriter.class, - "Write random binary key/value pairs")); - - // Make sure they gave us a program name. - if (args.length == 0) { - System.out.println("An example program must be given as the" + - " first argument."); - printUsage(programs); - return; - } - - // And that it is good. - ProgramDescription pgm = (ProgramDescription) programs.get(args[0]); - if (pgm == null) { - System.out.println("Unknown program '" + args[0] + "' chosen."); - printUsage(programs); - return; - } - - // Remove the leading argument and call main - String[] new_args = new String[args.length - 1]; - for(int i=1; i < args.length; ++i) { - new_args[i-1] = args[i]; - } - pgm.invoke(new_args); - } - } + Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ProgramDriver.java URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ProgramDriver.java?rev=399077&view=auto ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ProgramDriver.java (added) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ProgramDriver.java Tue May 2 16:07:32 2006 @@ -0,0 +1,144 @@ +/** + * Copyright 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.TreeMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; + +/** A driver that is used to run programs added to it + */ + +public class ProgramDriver { + + /** + * A description of a program based on its class and a + * human-readable description. + * @author Owen O'Malley + * @date april 2006 + */ + Map programs; + + public ProgramDriver(){ + programs = new TreeMap(); + } + + static private class ProgramDescription { + + static final Class[] paramTypes = new Class[] {String[].class}; + + /** + * Create a description of an example program. + * @param mainClass the class with the main for the example program + * @param description a string to display to the user in help messages + * @throws SecurityException if we can't use reflection + * @throws NoSuchMethodException if the class doesn't have a main method + */ + public ProgramDescription(Class mainClass, + String description) + throws SecurityException, NoSuchMethodException { + this.main = mainClass.getMethod("main", paramTypes); + this.description = description; + } + + /** + * Invoke the example application with the given arguments + * @param args the arguments for the application + * @throws Throwable The exception thrown by the invoked method + */ + public void invoke(String[] args) + throws Throwable { + try { + main.invoke(null, new Object[]{args}); + } catch (InvocationTargetException except) { + throw except.getCause(); + } + } + + public String getDescription() { + return description; + } + + private Method main; + private String description; + } + + private static void printUsage(Map programs) { + System.out.println("Valid program names are:"); + for(Iterator itr=programs.entrySet().iterator(); itr.hasNext();) { + Map.Entry item = (Entry) itr.next(); + System.out.println(" " + (String) item.getKey() + ": " + + ((ProgramDescription) item.getValue()).getDescription()); + } + } + + /** + * This is the method that adds the classed to the repository + * @param name The name of the string you want the class instance to be called with + * @param mainClass The class that you want to add to the repository + * @param description The description of the class + * @throws NoSuchMethodException + * @throws SecurityException + */ + public void addClass (String name, Class mainClass, String description) throws Throwable { + programs.put(name , new ProgramDescription(mainClass, description)); + } + + /** + * This is a driver for the example programs. + * It looks at the first command line argument and tries to find an + * example program with that name. + * If it is found, it calls the main method in that class with the rest + * of the command line arguments. + * @param args The argument from the user. args[0] is the command to run. + * @throws NoSuchMethodException + * @throws SecurityException + * @throws IllegalAccessException + * @throws IllegalArgumentException + * @throws Throwable Anything thrown by the example program's main + */ + public void driver(String[] args) + throws Throwable + { + // Make sure they gave us a program name. + if (args.length == 0) { + System.out.println("An example program must be given as the" + + " first argument."); + printUsage(programs); + return; + } + + // And that it is good. + ProgramDescription pgm = (ProgramDescription) programs.get(args[0]); + if (pgm == null) { + System.out.println("Unknown program '" + args[0] + "' chosen."); + printUsage(programs); + return; + } + + // Remove the leading argument and call main + String[] new_args = new String[args.length - 1]; + for(int i=1; i < args.length; ++i) { + new_args[i-1] = args[i]; + } + pgm.invoke(new_args); + } + +} Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java?rev=399077&r1=399076&r2=399077&view=diff ============================================================================== --- lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java (original) +++ lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java Tue May 2 16:07:32 2006 @@ -59,6 +59,10 @@ writeTest(fs, false); readTest(fs, false); seekTest(fs, false); + fs.delete(CONTROL_DIR); + fs.delete(DATA_DIR); + fs.delete(WRITE_DIR); + fs.delete(READ_DIR); } public static void createControlFile(FileSystem fs, Copied: lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMapRed.java (from r399074, lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MapredLoadTest.java) URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMapRed.java?p2=lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMapRed.java&p1=lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MapredLoadTest.java&r1=399074&r2=399077&rev=399077&view=diff ============================================================================== --- lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MapredLoadTest.java (original) +++ lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMapRed.java Tue May 2 16:07:32 2006 @@ -18,10 +18,11 @@ import org.apache.hadoop.fs.*; import org.apache.hadoop.io.*; import org.apache.hadoop.conf.*; - +import junit.framework.TestCase; import java.io.*; import java.util.*; + /********************************************************** * MapredLoadTest generates a bunch of work that exercises * a Hadoop Map-Reduce system (and DFS, too). It goes through @@ -55,8 +56,9 @@ * 7) A mapred job integrates all the count files into a single one. * **********************************************************/ -public class MapredLoadTest { +public class TestMapRed extends TestCase { /** + * Modified to make it a junit test. * The RandomGen Job does the actual work of creating * a huge file of assorted numbers. It receives instructions * as to how many times each number should be counted. Then @@ -197,24 +199,27 @@ } } - int range; - int counts; - Random r = new Random(); - Configuration conf; + private static int range = 10; + private static int counts = 100; + private static Random r = new Random(); + private static Configuration conf = new Configuration(); /** - * MapredLoadTest - */ - public MapredLoadTest(int range, int counts, Configuration conf) throws IOException { - this.range = range; - this.counts = counts; - this.conf = conf; + public TestMapRed(int range, int counts, Configuration conf) throws IOException { + this.range = range; + this.counts = counts; + this.conf = conf; + } + **/ + + public void testMapred() throws Exception { + launch(); } /** * */ - public void launch() throws IOException { + public static void launch() throws Exception { // // Generate distribution of ints. This is the answer key. // @@ -268,7 +273,7 @@ // file of random numbers. // Path randomOuts = new Path(testdir, "genouts"); - fs.mkdirs(randomOuts); + //fs.mkdirs(randomOuts); JobConf genJob = new JobConf(conf); @@ -317,7 +322,7 @@ // int intermediateReduces = 10; Path intermediateOuts = new Path(testdir, "intermediateouts"); - fs.mkdirs(intermediateOuts); + //fs.mkdirs(intermediateOuts); JobConf checkJob = new JobConf(conf); checkJob.setInputPath(randomOuts); checkJob.setInputKeyClass(LongWritable.class); @@ -342,7 +347,7 @@ // all the files. // Path finalOuts = new Path(testdir, "finalouts"); - fs.mkdirs(finalOuts); + //fs.mkdirs(finalOuts); JobConf mergeJob = new JobConf(conf); mergeJob.setInputPath(intermediateOuts); mergeJob.setInputKeyClass(IntWritable.class); @@ -415,6 +420,7 @@ } finally { bw.close(); } + fs.delete(testdir); } /** @@ -422,7 +428,7 @@ */ public static void main(String[] argv) throws Exception { if (argv.length < 2) { - System.err.println("Usage: MapredLoadTest <range> <counts>"); + System.err.println("Usage: TestMapRed <range> <counts>"); System.err.println(); System.err.println("Note: a good test will have a <counts> value that is substantially larger than the <range>"); return; @@ -431,8 +437,6 @@ int i = 0; int range = Integer.parseInt(argv[i++]); int counts = Integer.parseInt(argv[i++]); - - MapredLoadTest mlt = new MapredLoadTest(range, counts, new Configuration()); - mlt.launch(); + launch(); } } Added: lucene/hadoop/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java?rev=399077&view=auto ============================================================================== --- lucene/hadoop/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java (added) +++ lucene/hadoop/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java Tue May 2 16:07:32 2006 @@ -0,0 +1,63 @@ +/** + * Copyright 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.test; + +import org.apache.hadoop.util.ProgramDriver; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.TreeMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import org.apache.hadoop.mapred.TestMapRed; +import org.apache.hadoop.mapred.TestTextInputFormat; +import org.apache.hadoop.mapred.TestSequenceFileInputFormat; +import org.apache.hadoop.dfs.ClusterTestDFS; +import org.apache.hadoop.fs.TestFileSystem; +import org.apache.hadoop.io.TestArrayFile; +import org.apache.hadoop.io.TestSetFile; +import org.apache.hadoop.io.TestSequenceFile; +import org.apache.hadoop.ipc.TestIPC; +import org.apache.hadoop.ipc.TestRPC; + +public class AllTestDriver { + + /** + * A description of the test program for running all the tests using jar file + * @date April 2006 + */ + + public static void main(String argv[]){ + ProgramDriver pgd = new ProgramDriver(); + try { + pgd.addClass("mapredtest", TestMapRed.class, "A map/reduce test check."); + pgd.addClass("clustertestdfs", ClusterTestDFS.class, "A pseudo distributed test for DFS."); + pgd.addClass("testfilesystem", TestFileSystem.class, "A test for FileSystem read/write."); + pgd.addClass("testsequencefile", TestSequenceFile.class, "A test for flat files of binary key value pairs."); + pgd.addClass("testsetfile", TestSetFile.class, "A test for flat files of binary key/value pairs."); + pgd.addClass("testarrayfile", TestArrayFile.class, "A test for flat files of binary key/value pairs."); + pgd.addClass("testrpc", TestRPC.class, "A test for rpc."); + pgd.addClass("testipc", TestIPC.class, "A test for ipc."); + pgd.addClass("testsequencefileinputformat", TestSequenceFileInputFormat.class, "A test for sequence file input format."); + pgd.addClass("testtextinputformat", TestTextInputFormat.class, "A test for text input format."); + pgd.driver(argv); + } + catch(Throwable e){ + e.printStackTrace(); + } + } +}