Repository: incubator-blur Updated Branches: refs/heads/apache-blur-0.2 a1f57f4ee -> 4147f3588
Made some more adjustments to get the tests to pass. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/4147f358 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/4147f358 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/4147f358 Branch: refs/heads/apache-blur-0.2 Commit: 4147f358809814d5a8472b804244c697d6861b15 Parents: a1f57f4 Author: Aaron McCurry <[email protected]> Authored: Wed Jul 2 11:39:20 2014 -0400 Committer: Aaron McCurry <[email protected]> Committed: Wed Jul 2 11:39:20 2014 -0400 ---------------------------------------------------------------------- blur-mapred-hadoop1/pom.xml | 2 +- .../mapreduce/lib/BlurOutputFormatTest.java | 39 ++++++-------------- .../blur/mapreduce/lib/CsvBlurDriverTest.java | 14 +++++++ 3 files changed, 27 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4147f358/blur-mapred-hadoop1/pom.xml ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/pom.xml b/blur-mapred-hadoop1/pom.xml index 53bf1e3..bc65493 100644 --- a/blur-mapred-hadoop1/pom.xml +++ b/blur-mapred-hadoop1/pom.xml @@ -194,7 +194,7 @@ under the License. <groupId>org.apache.mrunit</groupId> <artifactId>mrunit</artifactId> <version>${mrunit.version}</version> - <classifier>hadoop1</classifier> + <classifier>hadoop2</classifier> <scope>test</scope> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4147f358/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java index 11ad0ce..96723c8 100644 --- a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java +++ b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java @@ -56,15 +56,14 @@ public class BlurOutputFormatTest { private static Configuration conf = new Configuration(); private static FileSystem localFs; private static MiniMRCluster mr; - private static Path TEST_ROOT_DIR; + private static final Path TEST_ROOT_DIR = new Path("./target/tmp/BlurOutputFormatTest_tmp"); private static JobConf jobConf; - private Path outDir = new Path(TEST_ROOT_DIR + "/out"); - private Path inDir = new Path(TEST_ROOT_DIR + "/in"); + private static final Path outDir = new Path(TEST_ROOT_DIR + "/out"); + private static final Path inDir = new Path(TEST_ROOT_DIR + "/in"); @BeforeClass public static void setupTest() throws Exception { System.setProperty("test.build.data", "./target/BlurOutputFormatTest/data"); - TEST_ROOT_DIR = new Path(System.getProperty("test.build.data", "target/tmp/BlurOutputFormatTest_tmp")); System.setProperty("hadoop.log.dir", "./target/BlurOutputFormatTest/hadoop_log"); try { localFs = FileSystem.getLocal(conf); @@ -97,18 +96,18 @@ public class BlurOutputFormatTest { } @Before - public void setup() { + public void setup() throws IllegalArgumentException, IOException { TableContext.clear(); + assertTrue(localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true)); + assertTrue(localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true)); } @Test public void testBlurOutputFormat() throws IOException, InterruptedException, ClassNotFoundException { - localFs.delete(inDir, true); - localFs.delete(outDir, true); writeRecordsFile("in/part1", 1, 1, 1, 1, "cf1"); writeRecordsFile("in/part2", 1, 1, 2, 1, "cf1"); - Job job = new Job(jobConf, "blur index"); + Job job = new Job(new Configuration(jobConf), "blur index"); job.setJarByClass(BlurOutputFormatTest.class); job.setMapperClass(CsvBlurMapper.class); job.setInputFormatClass(TextInputFormat.class); @@ -153,13 +152,10 @@ public class BlurOutputFormatTest { @Test public void testBlurOutputFormatOverFlowTest() throws IOException, InterruptedException, ClassNotFoundException { - localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true); - localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true); - writeRecordsFile("in/part1", 1, 50, 1, 1500, "cf1"); // 1500 * 50 = 75,000 writeRecordsFile("in/part2", 1, 50, 2000, 100, "cf1"); // 100 * 50 = 5,000 - Job job = new Job(jobConf, "blur index"); + Job job = new Job(new Configuration(jobConf), "blur index"); job.setJarByClass(BlurOutputFormatTest.class); job.setMapperClass(CsvBlurMapper.class); job.setInputFormatClass(TextInputFormat.class); @@ -195,13 +191,10 @@ public class BlurOutputFormatTest { @Test public void testBlurOutputFormatOverFlowMultipleReducersTest() throws IOException, InterruptedException, ClassNotFoundException { - localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true); - localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true); - writeRecordsFile("in/part1", 1, 50, 1, 1500, "cf1"); // 1500 * 50 = 75,000 writeRecordsFile("in/part2", 1, 50, 2000, 100, "cf1"); // 100 * 50 = 5,000 - Job job = new Job(jobConf, "blur index"); + Job job = new Job(new Configuration(jobConf), "blur index"); job.setJarByClass(BlurOutputFormatTest.class); job.setMapperClass(CsvBlurMapper.class); job.setInputFormatClass(TextInputFormat.class); @@ -243,13 +236,10 @@ public class BlurOutputFormatTest { @Test public void testBlurOutputFormatOverFlowMultipleReducersWithReduceMultiplierTest() throws IOException, InterruptedException, ClassNotFoundException { - localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true); - localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true); - writeRecordsFile("in/part1", 1, 50, 1, 1500, "cf1"); // 1500 * 50 = 75,000 writeRecordsFile("in/part2", 1, 50, 2000, 100, "cf1"); // 100 * 50 = 5,000 - Job job = new Job(jobConf, "blur index"); + Job job = new Job(new Configuration(jobConf), "blur index"); job.setJarByClass(BlurOutputFormatTest.class); job.setMapperClass(CsvBlurMapper.class); job.setInputFormatClass(TextInputFormat.class); @@ -291,12 +281,10 @@ public class BlurOutputFormatTest { @Test(expected = IllegalArgumentException.class) public void testBlurOutputFormatValidateReducerCount() throws IOException, InterruptedException, ClassNotFoundException { - localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true); - localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true); writeRecordsFile("in/part1", 1, 1, 1, 1, "cf1"); writeRecordsFile("in/part2", 1, 1, 2, 1, "cf1"); - Job job = new Job(jobConf, "blur index"); + Job job = new Job(new Configuration(jobConf), "blur index"); job.setJarByClass(BlurOutputFormatTest.class); job.setMapperClass(CsvBlurMapper.class); job.setInputFormatClass(TextInputFormat.class); @@ -323,14 +311,11 @@ public class BlurOutputFormatTest { // @Test public void testBlurOutputFormatCleanupDuringJobKillTest() throws IOException, InterruptedException, ClassNotFoundException { - localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true); - localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true); - writeRecordsFile("in/part1", 1, 50, 1, 1500, "cf1"); // 1500 * 50 = 75,000 writeRecordsFile("in/part2", 1, 5000, 2000, 100, "cf1"); // 100 * 5000 = // 500,000 - Job job = new Job(jobConf, "blur index"); + Job job = new Job(new Configuration(jobConf), "blur index"); job.setJarByClass(BlurOutputFormatTest.class); job.setMapperClass(CsvBlurMapper.class); job.setInputFormatClass(TextInputFormat.class); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4147f358/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java index ec3239e..6a673eb 100644 --- a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java +++ b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import java.io.IOException; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Proxy; @@ -31,8 +32,11 @@ import org.apache.blur.mapreduce.lib.CsvBlurDriver.ControllerPool; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.compress.SnappyCodec; import org.apache.hadoop.mapreduce.Job; +import org.junit.Before; import org.junit.Test; public class CsvBlurDriverTest { @@ -40,6 +44,16 @@ public class CsvBlurDriverTest { protected String tableUri = "file:///tmp/tmppath"; protected int shardCount = 13; + @Before + public void setup() throws IOException { + Configuration configuration = new Configuration(); + Path path1 = new Path("file:///tmp/test1"); + Path path2 = new Path("file:///tmp/test2"); + FileSystem fileSystem = path1.getFileSystem(configuration); + fileSystem.mkdirs(path1); + fileSystem.mkdirs(path2); + } + @Test public void testCsvBlurDriverTestFail1() throws Exception { Configuration configuration = new Configuration();
