Repository: crunch Updated Branches: refs/heads/master 9bb526fd1 -> 6c8c5ffd0
CRUNCH-524 Correctly delete temp files Delete /tmp/crunch- on exit; delete ~/hbase after test finishes; move H2 db file to /tmp Signed-off-by: Gabriel Reid <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/6c8c5ffd Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/6c8c5ffd Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/6c8c5ffd Branch: refs/heads/master Commit: 6c8c5ffd0447da82e63cf0b74ed2026c77a0c554 Parents: 9bb526f Author: Sean Owen <[email protected]> Authored: Thu May 21 22:34:16 2015 +0100 Committer: Gabriel Reid <[email protected]> Committed: Sat May 23 21:34:04 2015 +0200 ---------------------------------------------------------------------- .../crunch/contrib/io/jdbc/DataBaseSourceIT.java | 5 +++-- .../crunch/impl/dist/DistributedPipeline.java | 4 +++- .../apache/crunch/io/hbase/WordCountHBaseIT.java | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/6c8c5ffd/crunch-contrib/src/it/java/org/apache/crunch/contrib/io/jdbc/DataBaseSourceIT.java ---------------------------------------------------------------------- diff --git a/crunch-contrib/src/it/java/org/apache/crunch/contrib/io/jdbc/DataBaseSourceIT.java b/crunch-contrib/src/it/java/org/apache/crunch/contrib/io/jdbc/DataBaseSourceIT.java index 1c48559..a19803e 100644 --- a/crunch-contrib/src/it/java/org/apache/crunch/contrib/io/jdbc/DataBaseSourceIT.java +++ b/crunch-contrib/src/it/java/org/apache/crunch/contrib/io/jdbc/DataBaseSourceIT.java @@ -46,12 +46,13 @@ public class DataBaseSourceIT extends CrunchTestSupport implements Serializable public void start() throws Exception { File file = tempDir.copyResourceFile("data.script"); server = Server.createTcpServer().start(); - RunScript.execute("jdbc:h2:tcp://localhost/~/test", "sa", "", file.getAbsolutePath(), "utf-8", false); + RunScript.execute("jdbc:h2:file:/tmp/test", "sa", "", file.getAbsolutePath(), "utf-8", false); } @After public void stop() throws Exception { server.stop(); + new File("/tmp/test.h2.db").delete(); } @Test @@ -59,7 +60,7 @@ public class DataBaseSourceIT extends CrunchTestSupport implements Serializable Pipeline pipeline = new MRPipeline(DataBaseSourceIT.class); DataBaseSource<IdentifiableName> dbsrc = new DataBaseSource.Builder<IdentifiableName>(IdentifiableName.class) .setDriverClass(org.h2.Driver.class) - .setUrl("jdbc:h2:tcp://localhost/~/test").setUsername("sa").setPassword("") + .setUrl("jdbc:h2:file:/tmp/test").setUsername("sa").setPassword("") .selectSQLQuery("SELECT ID, NAME FROM TEST").countSQLQuery("select count(*) from Test").build(); PCollection<IdentifiableName> cdidata = pipeline.read(dbsrc); http://git-wip-us.apache.org/repos/asf/crunch/blob/6c8c5ffd/crunch-core/src/main/java/org/apache/crunch/impl/dist/DistributedPipeline.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/dist/DistributedPipeline.java b/crunch-core/src/main/java/org/apache/crunch/impl/dist/DistributedPipeline.java index 88da5a6..049046a 100644 --- a/crunch-core/src/main/java/org/apache/crunch/impl/dist/DistributedPipeline.java +++ b/crunch-core/src/main/java/org/apache/crunch/impl/dist/DistributedPipeline.java @@ -396,7 +396,9 @@ public abstract class DistributedPipeline implements Pipeline { private static Path createTempDirectory(Configuration conf) { Path dir = createTemporaryPath(conf); try { - dir.getFileSystem(conf).mkdirs(dir); + FileSystem fs = dir.getFileSystem(conf); + fs.mkdirs(dir); + fs.deleteOnExit(dir); } catch (IOException e) { throw new RuntimeException("Cannot create job output directory " + dir, e); } http://git-wip-us.apache.org/repos/asf/crunch/blob/6c8c5ffd/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/WordCountHBaseIT.java ---------------------------------------------------------------------- diff --git a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/WordCountHBaseIT.java b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/WordCountHBaseIT.java index dd48352..ead2aea 100644 --- a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/WordCountHBaseIT.java +++ b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/WordCountHBaseIT.java @@ -19,6 +19,7 @@ package org.apache.crunch.io.hbase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.IOException; @@ -39,6 +40,8 @@ import org.apache.crunch.test.TemporaryPath; import org.apache.crunch.test.TemporaryPaths; import org.apache.crunch.types.writable.Writables; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.client.Delete; @@ -119,6 +122,18 @@ public class WordCountHBaseIT { @Before public void setUp() throws Exception { Configuration conf = HBaseConfiguration.create(tmpDir.getDefaultConfiguration()); + + // HBase 0.98 HBaseTestingUtility hard-codes ~/hbase as its temp directory and leaves + // it lying around after this test. It's easy to delete, but to be sure it's safe to + // delete, we need to be sure that it doesn't exist already. The test probably won't + // work anyway if it does. If it doesn't exist, then deleting it later doesn't matter + // even if a future HBase is used (which doesn't have this problem) and this test + // isn't updated. + FileSystem fs = FileSystem.get(conf); + Path hbaseTestPath = new Path(fs.makeQualified(fs.getHomeDirectory()),"hbase"); + assertFalse("The hbase directory in your home directory already exists. This test deletes this directory " + + "on exit, so it will not run unless you delete the directory yourself", fs.exists(hbaseTestPath)); + hbaseTestUtil = new HBaseTestingUtility(conf); hbaseTestUtil.startMiniZKCluster(); hbaseTestUtil.startMiniHBaseCluster(1, 1); @@ -133,6 +148,9 @@ public class WordCountHBaseIT { public void tearDown() throws Exception { hbaseTestUtil.shutdownMiniHBaseCluster(); hbaseTestUtil.shutdownMiniZKCluster(); + FileSystem fs = FileSystem.get(hbaseTestUtil.getConfiguration()); + Path hbaseTestPath = new Path(fs.makeQualified(fs.getHomeDirectory()),"hbase"); + fs.delete(hbaseTestPath, true); } public void run(Pipeline pipeline) throws Exception {
