http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9c42c0b3/crunch/src/it/java/org/apache/crunch/lib/join/MultiAvroSchemaJoinIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/lib/join/MultiAvroSchemaJoinIT.java b/crunch/src/it/java/org/apache/crunch/lib/join/MultiAvroSchemaJoinIT.java index 24427bd..e9b2fa2 100644 --- a/crunch/src/it/java/org/apache/crunch/lib/join/MultiAvroSchemaJoinIT.java +++ b/crunch/src/it/java/org/apache/crunch/lib/join/MultiAvroSchemaJoinIT.java @@ -38,7 +38,7 @@ import org.apache.crunch.io.From; import org.apache.crunch.test.Employee; import org.apache.crunch.test.Person; import org.apache.crunch.test.TemporaryPath; -import org.apache.hadoop.conf.Configuration; +import org.apache.crunch.test.TemporaryPaths; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -52,7 +52,7 @@ public class MultiAvroSchemaJoinIT { private File personFile; private File employeeFile; @Rule - public TemporaryPath temporaryPath= new TemporaryPath(); + public TemporaryPath tmpDir = TemporaryPaths.create(); @Before public void setUp() throws Exception { @@ -107,7 +107,7 @@ public class MultiAvroSchemaJoinIT { @Test public void testJoin() throws Exception { - Pipeline p = new MRPipeline(MultiAvroSchemaJoinIT.class, temporaryPath.setTempLoc(new Configuration())); + Pipeline p = new MRPipeline(MultiAvroSchemaJoinIT.class, tmpDir.getDefaultConfiguration()); PCollection<Person> people = p.read(From.avroFile(personFile.getAbsolutePath(), records(Person.class))); PCollection<Employee> employees = p.read(From.avroFile(employeeFile.getAbsolutePath(), records(Employee.class)));
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9c42c0b3/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java b/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java deleted file mode 100644 index e96e3b1..0000000 --- a/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java +++ /dev/null @@ -1,113 +0,0 @@ -package org.apache.crunch.test; - -import java.io.File; -import java.io.IOException; - -import org.apache.crunch.impl.mr.run.RuntimeParameters; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.junit.rules.ExternalResource; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.Description; -import org.junit.runners.model.Statement; - -import com.google.common.io.Files; -import com.google.common.io.Resources; - -/** - * Creates a temporary directory for a test case and destroys it afterwards. - * This provides a temporary directory like JUnit's {@link TemporaryFolder} but - * geared towards Hadoop applications. Unlike {@link TemporaryFolder}, it - * doesn't create any files or directories except for the root directory itself. - */ -public final class TemporaryPath extends ExternalResource { - private TemporaryFolder tmp = new TemporaryFolder(); - - @Override - public Statement apply(Statement base, Description description) { - return tmp.apply(base, description); - } - - /** - * Get the root directory which will be deleted automatically. - */ - public File getRootFile() { - return tmp.getRoot(); - } - - /** - * Get the root directory as a {@link Path}. - */ - public Path getRootPath() { - return toPath(tmp.getRoot()); - } - - /** - * Get the root directory as an absolute file name. - */ - public String getRootFileName() { - return tmp.getRoot().getAbsolutePath(); - } - - /** - * Get a {@link File} below the temporary directory. - */ - public File getFile(String fileName) { - return new File(getRootFile(), fileName); - } - - /** - * Get a {@link Path} below the temporary directory. - */ - public Path getPath(String fileName) { - return toPath(getFile(fileName)); - } - - /** - * Get an absolute file name below the temporary directory. - */ - public String getFileName(String fileName) { - return getFile(fileName).getAbsolutePath(); - } - - /** - * Copy a classpath resource to {@link File}. - */ - public File copyResourceFile(String resourceName) throws IOException { - File dest = new File(tmp.getRoot(), resourceName); - copy(resourceName, dest); - return dest; - } - - /** - * Copy a classpath resource to a {@link Path} - */ - public Path copyResourcePath(String resourceName) throws IOException { - return toPath(copyResourceFile(resourceName)); - } - - /** - * Copy a classpath resource returning its absolute file name. - */ - public String copyResourceFileName(String resourceName) throws IOException { - return copyResourceFile(resourceName).getAbsolutePath(); - } - - private void copy(String resourceName, File dest) throws IOException { - Files.copy(Resources.newInputStreamSupplier(Resources.getResource(resourceName)), dest); - } - - private static Path toPath(File file) { - return new Path(file.getAbsolutePath()); - } - - public Configuration getDefaultConfiguration() throws IOException { - return setTempLoc(new Configuration()); - } - - public Configuration setTempLoc(Configuration config) throws IOException { - config.set(RuntimeParameters.TMP_DIR, getRootFileName()); - config.set("hadoop.tmp.dir", getFileName("hadoop-tmp")); - return config; - } -} http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9c42c0b3/crunch/src/it/java/org/apache/crunch/test/TemporaryPathIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/test/TemporaryPathIT.java b/crunch/src/it/java/org/apache/crunch/test/TemporaryPathIT.java deleted file mode 100644 index 65018da..0000000 --- a/crunch/src/it/java/org/apache/crunch/test/TemporaryPathIT.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.apache.crunch.test; - -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; - -import java.io.File; -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocalFileSystem; -import org.junit.Rule; -import org.junit.Test; - - -public class TemporaryPathIT { - @Rule - public TemporaryPath tmpDir = new TemporaryPath(); - - @Test - public void testRoot() throws IOException { - assertThat(tmpDir.getRootFile().exists(), is(true)); - assertThat(new File(tmpDir.getRootFileName()).exists(), is(true)); - assertThat(getFs().exists(tmpDir.getRootPath()), is(true)); - } - - @Test - public void testFile() throws IOException { - assertThat(tmpDir.getFile("foo").getParentFile(), is(tmpDir.getRootFile())); - assertThat(tmpDir.getFile("foo").getName(), is("foo")); - assertThat(tmpDir.getFile("foo").exists(), is(false)); - } - - @Test - public void testPath() throws IOException { - assertThat(tmpDir.getPath("foo").getParent(), is(tmpDir.getRootPath())); - assertThat(tmpDir.getPath("foo").getName(), is("foo")); - assertThat(getFs().exists(tmpDir.getPath("foo")), is(false)); - } - - @Test - public void testFileName() { - assertThat(new File(tmpDir.getRootFileName()), is(tmpDir.getRootFile())); - assertThat(new File(tmpDir.getFileName("foo").toString()), is(tmpDir.getFile("foo"))); - } - - @Test - public void testCopyResource() throws IOException { - File dest = tmpDir.getFile("shakes.txt"); - assertThat(dest.exists(), is(false)); - - tmpDir.copyResourceFile("shakes.txt"); - assertThat(dest.exists(), is(true)); - } - - private LocalFileSystem getFs() throws IOException { - return FileSystem.getLocal(new Configuration()); - } -} http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9c42c0b3/crunch/src/it/java/org/apache/crunch/test/TemporaryPaths.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/test/TemporaryPaths.java b/crunch/src/it/java/org/apache/crunch/test/TemporaryPaths.java new file mode 100644 index 0000000..af8cf23 --- /dev/null +++ b/crunch/src/it/java/org/apache/crunch/test/TemporaryPaths.java @@ -0,0 +1,23 @@ +package org.apache.crunch.test; + +import org.apache.crunch.impl.mr.run.RuntimeParameters; +import org.apache.hadoop.conf.Configuration; + + +/** + * Utilities for working with {@link TemporaryPath}. + */ +public final class TemporaryPaths { + + /** + * Static factory returning a {@link TemporaryPath} with adjusted + * {@link Configuration} properties. + */ + public static TemporaryPath create() { + return new TemporaryPath(RuntimeParameters.TMP_DIR, "hadoop.tmp.dir"); + } + + private TemporaryPaths() { + // nothing + } +} http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9c42c0b3/crunch/src/main/java/org/apache/crunch/test/FileHelper.java ---------------------------------------------------------------------- diff --git a/crunch/src/main/java/org/apache/crunch/test/FileHelper.java b/crunch/src/main/java/org/apache/crunch/test/FileHelper.java deleted file mode 100644 index ca5a4b3..0000000 --- a/crunch/src/main/java/org/apache/crunch/test/FileHelper.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.crunch.test; - -import static com.google.common.io.Resources.getResource; -import static com.google.common.io.Resources.newInputStreamSupplier; - -import java.io.File; -import java.io.IOException; - -import com.google.common.io.Files; - -public class FileHelper { - - public static String createTempCopyOf(String fileResource) throws IOException { - File tmpFile = File.createTempFile("tmp", ""); - tmpFile.deleteOnExit(); - Files.copy(newInputStreamSupplier(getResource(fileResource)), tmpFile); - return tmpFile.getAbsolutePath(); - } - - public static File createOutputPath() throws IOException { - File output = File.createTempFile("output", ""); - output.delete(); - return output; - } -} http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9c42c0b3/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index afa73f4..a0d7c0d 100644 --- a/pom.xml +++ b/pom.xml @@ -34,12 +34,14 @@ under the License. <modules> <module>crunch</module> + <module>crunch-test</module> <module>examples</module> <module>scrunch</module> </modules> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <java.source.version>1.6</java.source.version> <java.target.version>1.6</java.target.version> <scala.version>2.9.2</scala.version> @@ -107,8 +109,7 @@ under the License. <dependency> <groupId>org.apache.crunch</groupId> - <artifactId>crunch</artifactId> - <type>test-jar</type> + <artifactId>crunch-test</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9c42c0b3/scrunch/pom.xml ---------------------------------------------------------------------- diff --git a/scrunch/pom.xml b/scrunch/pom.xml index 0b549ae..33e4119 100644 --- a/scrunch/pom.xml +++ b/scrunch/pom.xml @@ -61,19 +61,13 @@ under the License. <artifactId>slf4j-log4j12</artifactId> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.scalatest</groupId> <artifactId>scalatest_${scala.version}</artifactId> <scope>test</scope> </dependency> <dependency> <groupId>org.apache.crunch</groupId> - <artifactId>crunch</artifactId> - <type>test-jar</type> + <artifactId>crunch-test</artifactId> <scope>test</scope> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9c42c0b3/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala ---------------------------------------------------------------------- diff --git a/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala b/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala index de012c7..d418189 100644 --- a/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala +++ b/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala @@ -41,7 +41,7 @@ class PipelineAppTest extends ScrunchTestSupport with JUnitSuite { args(0) = tempDir.copyResourceFileName("shakes.txt") args(1) = tempDir.copyResourceFileName("maugham.txt") args(2) = tempDir.getFileName("output") - tempDir.setTempLoc(WordCount.configuration) + tempDir.overridePathProperties(WordCount.configuration) WordCount.main(args) } }
