CRUNCH-26: Get rid of FileHelper. Replace FileHelper with TemporaryPath and remove FileHelper. Fix left-over files in Scrunch integration tests.
Signed-off-by: jwills <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/47a9d1c4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/47a9d1c4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/47a9d1c4 Branch: refs/heads/master Commit: 47a9d1c456663021903d77116c65d5723fe8ede6 Parents: 9c42c0b Author: Matthias Friedrich <[email protected]> Authored: Sat Jul 28 17:03:16 2012 +0200 Committer: jwills <[email protected]> Committed: Sat Jul 28 11:17:59 2012 -0700 ---------------------------------------------------------------------- .../java/org/apache/crunch/test/FileHelper.java | 42 --------------- .../src/it/java/org/apache/crunch/PageRankIT.java | 5 +- .../src/it/java/org/apache/crunch/lib/SortIT.java | 13 ++--- .../org/apache/crunch/lib/join/MapsideJoinIT.java | 5 +- .../org/apache/scrunch/ScrunchTestSupport.java | 3 +- 5 files changed, 12 insertions(+), 56 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/47a9d1c4/crunch-test/src/main/java/org/apache/crunch/test/FileHelper.java ---------------------------------------------------------------------- diff --git a/crunch-test/src/main/java/org/apache/crunch/test/FileHelper.java b/crunch-test/src/main/java/org/apache/crunch/test/FileHelper.java deleted file mode 100644 index ca5a4b3..0000000 --- a/crunch-test/src/main/java/org/apache/crunch/test/FileHelper.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.crunch.test; - -import static com.google.common.io.Resources.getResource; -import static com.google.common.io.Resources.newInputStreamSupplier; - -import java.io.File; -import java.io.IOException; - -import com.google.common.io.Files; - -public class FileHelper { - - public static String createTempCopyOf(String fileResource) throws IOException { - File tmpFile = File.createTempFile("tmp", ""); - tmpFile.deleteOnExit(); - Files.copy(newInputStreamSupplier(getResource(fileResource)), tmpFile); - return tmpFile.getAbsolutePath(); - } - - public static File createOutputPath() throws IOException { - File output = File.createTempFile("output", ""); - output.delete(); - return output; - } -} http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/47a9d1c4/crunch/src/it/java/org/apache/crunch/PageRankIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/PageRankIT.java b/crunch/src/it/java/org/apache/crunch/PageRankIT.java index fc0640a..e5fbfe6 100644 --- a/crunch/src/it/java/org/apache/crunch/PageRankIT.java +++ b/crunch/src/it/java/org/apache/crunch/PageRankIT.java @@ -25,7 +25,6 @@ import java.util.List; import org.apache.crunch.impl.mem.MemPipeline; import org.apache.crunch.impl.mr.MRPipeline; import org.apache.crunch.lib.Aggregate; -import org.apache.crunch.test.FileHelper; import org.apache.crunch.test.TemporaryPath; import org.apache.crunch.test.TemporaryPaths; import org.apache.crunch.types.PType; @@ -128,8 +127,8 @@ public class PageRankIT { }, input.getPTableType()); } - public static void run(Pipeline pipeline, PType<PageRankData> prType, PTypeFamily ptf) throws Exception { - String urlInput = FileHelper.createTempCopyOf("urls.txt"); + public void run(Pipeline pipeline, PType<PageRankData> prType, PTypeFamily ptf) throws Exception { + String urlInput = tmpDir.copyResourceFileName("urls.txt"); PTable<String, PageRankData> scores = pipeline.readTextFile(urlInput) .parallelDo(new MapFn<String, Pair<String, String>>() { @Override http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/47a9d1c4/crunch/src/it/java/org/apache/crunch/lib/SortIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/lib/SortIT.java b/crunch/src/it/java/org/apache/crunch/lib/SortIT.java index dbc0c7e..dc025a7 100644 --- a/crunch/src/it/java/org/apache/crunch/lib/SortIT.java +++ b/crunch/src/it/java/org/apache/crunch/lib/SortIT.java @@ -41,7 +41,6 @@ import org.apache.crunch.TupleN; import org.apache.crunch.impl.mr.MRPipeline; import org.apache.crunch.lib.Sort.ColumnOrder; import org.apache.crunch.lib.Sort.Order; -import org.apache.crunch.test.FileHelper; import org.apache.crunch.test.StringWrapper; import org.apache.crunch.test.TemporaryPath; import org.apache.crunch.test.TemporaryPaths; @@ -196,7 +195,7 @@ public class SortIT implements Serializable { } private void runSingle(Pipeline pipeline, PTypeFamily typeFamily, Order order, String firstLine) throws IOException { - String inputPath = FileHelper.createTempCopyOf("docs.txt"); + String inputPath = tmpDir.copyResourceFileName("docs.txt"); PCollection<String> input = pipeline.readTextFile(inputPath); // following turns the input from Writables to required type family @@ -215,7 +214,7 @@ public class SortIT implements Serializable { private void runPair(Pipeline pipeline, PTypeFamily typeFamily, ColumnOrder first, ColumnOrder second, String firstField, String secondField) throws IOException { - String inputPath = FileHelper.createTempCopyOf("docs.txt"); + String inputPath = tmpDir.copyResourceFileName("docs.txt"); PCollection<String> input = pipeline.readTextFile(inputPath); PCollection<Pair<String, String>> kv = input.parallelDo(new DoFn<String, Pair<String, String>>() { @@ -235,7 +234,7 @@ public class SortIT implements Serializable { private void runTriple(Pipeline pipeline, PTypeFamily typeFamily, ColumnOrder first, ColumnOrder second, ColumnOrder third, String firstField, String secondField, String thirdField) throws IOException { - String inputPath = FileHelper.createTempCopyOf("docs.txt"); + String inputPath = tmpDir.copyResourceFileName("docs.txt"); PCollection<String> input = pipeline.readTextFile(inputPath); PCollection<Tuple3<String, String, String>> kv = input.parallelDo( @@ -259,7 +258,7 @@ public class SortIT implements Serializable { private void runQuad(Pipeline pipeline, PTypeFamily typeFamily, ColumnOrder first, ColumnOrder second, ColumnOrder third, ColumnOrder fourth, String firstField, String secondField, String thirdField, String fourthField) throws IOException { - String inputPath = FileHelper.createTempCopyOf("docs.txt"); + String inputPath = tmpDir.copyResourceFileName("docs.txt"); PCollection<String> input = pipeline.readTextFile(inputPath); PCollection<Tuple4<String, String, String, String>> kv = input.parallelDo( @@ -283,7 +282,7 @@ public class SortIT implements Serializable { private void runTupleN(Pipeline pipeline, PTypeFamily typeFamily, ColumnOrder[] orders, String[] fields) throws IOException { - String inputPath = FileHelper.createTempCopyOf("docs.txt"); + String inputPath = tmpDir.copyResourceFileName("docs.txt"); PCollection<String> input = pipeline.readTextFile(inputPath); PType[] types = new PType[orders.length]; @@ -306,7 +305,7 @@ public class SortIT implements Serializable { } private void runTable(Pipeline pipeline, PTypeFamily typeFamily, String firstKey) throws IOException { - String inputPath = FileHelper.createTempCopyOf("docs.txt"); + String inputPath = tmpDir.copyResourceFileName("docs.txt"); PCollection<String> input = pipeline.readTextFile(inputPath); PTable<String, String> table = input.parallelDo(new DoFn<String, Pair<String, String>>() { http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/47a9d1c4/crunch/src/it/java/org/apache/crunch/lib/join/MapsideJoinIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/lib/join/MapsideJoinIT.java b/crunch/src/it/java/org/apache/crunch/lib/join/MapsideJoinIT.java index 25472fb..9982ba4 100644 --- a/crunch/src/it/java/org/apache/crunch/lib/join/MapsideJoinIT.java +++ b/crunch/src/it/java/org/apache/crunch/lib/join/MapsideJoinIT.java @@ -32,7 +32,6 @@ import org.apache.crunch.Pipeline; import org.apache.crunch.impl.mem.MemPipeline; import org.apache.crunch.impl.mr.MRPipeline; import org.apache.crunch.impl.mr.run.CrunchRuntimeException; -import org.apache.crunch.test.FileHelper; import org.apache.crunch.test.TemporaryPath; import org.apache.crunch.test.TemporaryPaths; import org.apache.crunch.types.writable.Writables; @@ -130,9 +129,9 @@ public class MapsideJoinIT { assertEquals(expectedJoinResult, joinedResultList); } - private static PTable<Integer, String> readTable(Pipeline pipeline, String filename) { + private PTable<Integer, String> readTable(Pipeline pipeline, String filename) { try { - return pipeline.readTextFile(FileHelper.createTempCopyOf(filename)).parallelDo("asTable", new LineSplitter(), + return pipeline.readTextFile(tmpDir.copyResourceFileName(filename)).parallelDo("asTable", new LineSplitter(), Writables.tableOf(Writables.ints(), Writables.strings())); } catch (IOException e) { throw new RuntimeException(e); http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/47a9d1c4/scrunch/src/it/java/org/apache/scrunch/ScrunchTestSupport.java ---------------------------------------------------------------------- diff --git a/scrunch/src/it/java/org/apache/scrunch/ScrunchTestSupport.java b/scrunch/src/it/java/org/apache/scrunch/ScrunchTestSupport.java index a9f7318..53d7cca 100644 --- a/scrunch/src/it/java/org/apache/scrunch/ScrunchTestSupport.java +++ b/scrunch/src/it/java/org/apache/scrunch/ScrunchTestSupport.java @@ -17,11 +17,12 @@ */ package org.apache.scrunch; +import org.apache.crunch.impl.mr.run.RuntimeParameters; import org.apache.crunch.test.TemporaryPath; import org.junit.Rule; public class ScrunchTestSupport { @Rule - public TemporaryPath tempDir = new TemporaryPath(); + public TemporaryPath tempDir = new TemporaryPath(RuntimeParameters.TMP_DIR, "hadoop.tmp.dir"); }
