Repository: incubator-beam Updated Branches: refs/heads/master 26a30a22d -> 6d0c205a3
[BEAM-1034] Clean up tmp area in tests Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ef74e192 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ef74e192 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ef74e192 Branch: refs/heads/master Commit: ef74e192eaee79e4cb8c7c901a296dd76559d76d Parents: 26a30a2 Author: Daniel Kulp <[email protected]> Authored: Tue Nov 22 13:31:19 2016 -0500 Committer: Jean-Baptiste Onofré <[email protected]> Committed: Thu Nov 24 07:55:58 2016 +0100 ---------------------------------------------------------------------- .../sorter/BufferedExternalSorter.java | 6 +- .../sdk/extensions/sorter/ExternalSorter.java | 6 +- .../sorter/BufferedExternalSorterTest.java | 58 +++++++++++++++++--- .../extensions/sorter/ExternalSorterTest.java | 53 +++++++++++++++--- 4 files changed, 103 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ef74e192/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorter.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorter.java b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorter.java index 0f89e30..1dfd339 100644 --- a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorter.java +++ b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorter.java @@ -35,12 +35,13 @@ public class BufferedExternalSorter implements Sorter { private int memoryMB = 100; /** Sets the path to a temporary location where the sorter writes intermediate files. */ - public void setTempLocation(String tempLocation) { + public Options setTempLocation(String tempLocation) { checkArgument( !tempLocation.startsWith("gs://"), "BufferedExternalSorter does not support GCS temporary location"); this.tempLocation = tempLocation; + return this; } /** Returns the configured temporary location. */ @@ -52,9 +53,10 @@ public class BufferedExternalSorter implements Sorter { * Sets the size of the memory buffer in megabytes. This controls both the buffer for initial in * memory sorting and the buffer used when external sorting. Must be greater than zero. */ - public void setMemoryMB(int memoryMB) { + public Options setMemoryMB(int memoryMB) { checkArgument(memoryMB > 0, "memoryMB must be greater than zero"); this.memoryMB = memoryMB; + return this; } /** Returns the configured size of the memory buffer. */ http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ef74e192/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/ExternalSorter.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/ExternalSorter.java b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/ExternalSorter.java index 3cf0cc0..beef1ee 100644 --- a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/ExternalSorter.java +++ b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/ExternalSorter.java @@ -67,12 +67,13 @@ class ExternalSorter implements Sorter { private int memoryMB = 100; /** Sets the path to a temporary location where the sorter writes intermediate files. */ - public void setTempLocation(String tempLocation) { + public Options setTempLocation(String tempLocation) { if (tempLocation.startsWith("gs://")) { throw new IllegalArgumentException("Sorter doesn't support GCS temporary location."); } this.tempLocation = tempLocation; + return this; } /** Returns the configured temporary location. */ @@ -81,9 +82,10 @@ class ExternalSorter implements Sorter { } /** Sets the size of the memory buffer in megabytes. */ - public void setMemoryMB(int memoryMB) { + public Options setMemoryMB(int memoryMB) { checkArgument(memoryMB > 0, "memoryMB must be greater than zero"); this.memoryMB = memoryMB; + return this; } /** Returns the configured size of the memory buffer. */ http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ef74e192/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorterTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorterTest.java b/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorterTest.java index 63dbedf..8c108eb 100644 --- a/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorterTest.java +++ b/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorterTest.java @@ -27,9 +27,17 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.Arrays; import org.apache.beam.sdk.extensions.sorter.SorterTestUtils.SorterGenerator; import org.apache.beam.sdk.values.KV; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -40,6 +48,29 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) public class BufferedExternalSorterTest { @Rule public ExpectedException thrown = ExpectedException.none(); + static Path tmpLocation; + + @BeforeClass + public static void setupTempDir() throws IOException { + tmpLocation = Files.createTempDirectory("tmp"); + } + + @AfterClass + public static void cleanupTempDir() throws IOException { + Files.walkFileTree(tmpLocation, new SimpleFileVisitor<Path>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } @SuppressWarnings("unchecked") @Test @@ -106,25 +137,29 @@ public class BufferedExternalSorterTest { @Test public void testEmpty() throws Exception { - SorterTestUtils.testEmpty(BufferedExternalSorter.create(new BufferedExternalSorter.Options())); + SorterTestUtils.testEmpty(BufferedExternalSorter.create(new BufferedExternalSorter.Options() + .setTempLocation(tmpLocation.toString()))); } @Test public void testSingleElement() throws Exception { SorterTestUtils.testSingleElement( - BufferedExternalSorter.create(new BufferedExternalSorter.Options())); + BufferedExternalSorter.create(new BufferedExternalSorter.Options() + .setTempLocation(tmpLocation.toString()))); } @Test public void testEmptyKeyValueElement() throws Exception { SorterTestUtils.testEmptyKeyValueElement( - BufferedExternalSorter.create(new BufferedExternalSorter.Options())); + BufferedExternalSorter.create(new BufferedExternalSorter.Options() + .setTempLocation(tmpLocation.toString()))); } @Test public void testMultipleIterations() throws Exception { SorterTestUtils.testMultipleIterations( - BufferedExternalSorter.create(new BufferedExternalSorter.Options())); + BufferedExternalSorter.create(new BufferedExternalSorter.Options() + .setTempLocation(tmpLocation.toString()))); } @Test @@ -133,7 +168,8 @@ public class BufferedExternalSorterTest { new SorterGenerator() { @Override public Sorter generateSorter() throws Exception { - return BufferedExternalSorter.create(new BufferedExternalSorter.Options()); + return BufferedExternalSorter.create(new BufferedExternalSorter.Options() + .setTempLocation(tmpLocation.toString())); } }, 1000000, @@ -146,7 +182,8 @@ public class BufferedExternalSorterTest { new SorterGenerator() { @Override public Sorter generateSorter() throws Exception { - return BufferedExternalSorter.create(new BufferedExternalSorter.Options()); + return BufferedExternalSorter.create(new BufferedExternalSorter.Options() + .setTempLocation(tmpLocation.toString())); } }, 1, @@ -156,14 +193,16 @@ public class BufferedExternalSorterTest { @Test public void testAddAfterSort() throws Exception { SorterTestUtils.testAddAfterSort( - BufferedExternalSorter.create(new BufferedExternalSorter.Options()), thrown); + BufferedExternalSorter.create(new BufferedExternalSorter.Options() + .setTempLocation(tmpLocation.toString())), thrown); fail(); } @Test public void testSortTwice() throws Exception { SorterTestUtils.testSortTwice( - BufferedExternalSorter.create(new BufferedExternalSorter.Options()), thrown); + BufferedExternalSorter.create(new BufferedExternalSorter.Options() + .setTempLocation(tmpLocation.toString())), thrown); fail(); } @@ -171,7 +210,8 @@ public class BufferedExternalSorterTest { public void testNegativeMemory() throws Exception { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("memoryMB must be greater than zero"); - BufferedExternalSorter.Options options = new BufferedExternalSorter.Options(); + BufferedExternalSorter.Options options = new BufferedExternalSorter.Options() + .setTempLocation(tmpLocation.toString()); options.setMemoryMB(-1); } } http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ef74e192/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/ExternalSorterTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/ExternalSorterTest.java b/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/ExternalSorterTest.java index 9232b62..bcfbdad 100644 --- a/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/ExternalSorterTest.java +++ b/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/ExternalSorterTest.java @@ -20,7 +20,16 @@ package org.apache.beam.sdk.extensions.sorter; import static org.junit.Assert.fail; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; + import org.apache.beam.sdk.extensions.sorter.SorterTestUtils.SorterGenerator; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -31,25 +40,52 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) public class ExternalSorterTest { @Rule public ExpectedException thrown = ExpectedException.none(); + static Path tmpLocation; + + @BeforeClass + public static void setupTempDir() throws IOException { + tmpLocation = Files.createTempDirectory("tmp"); + } + + @AfterClass + public static void cleanupTempDir() throws IOException { + Files.walkFileTree(tmpLocation, new SimpleFileVisitor<Path>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } @Test public void testEmpty() throws Exception { - SorterTestUtils.testEmpty(ExternalSorter.create(new ExternalSorter.Options())); + SorterTestUtils.testEmpty(ExternalSorter.create(new ExternalSorter.Options() + .setTempLocation(tmpLocation.toString()))); } @Test public void testSingleElement() throws Exception { - SorterTestUtils.testSingleElement(ExternalSorter.create(new ExternalSorter.Options())); + SorterTestUtils.testSingleElement(ExternalSorter.create(new ExternalSorter.Options() + .setTempLocation(tmpLocation.toString()))); } @Test public void testEmptyKeyValueElement() throws Exception { - SorterTestUtils.testEmptyKeyValueElement(ExternalSorter.create(new ExternalSorter.Options())); + SorterTestUtils.testEmptyKeyValueElement(ExternalSorter.create(new ExternalSorter.Options() + .setTempLocation(tmpLocation.toString()))); } @Test public void testMultipleIterations() throws Exception { - SorterTestUtils.testMultipleIterations(ExternalSorter.create(new ExternalSorter.Options())); + SorterTestUtils.testMultipleIterations(ExternalSorter.create(new ExternalSorter.Options() + .setTempLocation(tmpLocation.toString()))); } @Test @@ -58,7 +94,8 @@ public class ExternalSorterTest { new SorterGenerator() { @Override public Sorter generateSorter() throws Exception { - return ExternalSorter.create(new ExternalSorter.Options()); + return ExternalSorter.create(new ExternalSorter.Options() + .setTempLocation(tmpLocation.toString())); } }, 1, @@ -67,13 +104,15 @@ public class ExternalSorterTest { @Test public void testAddAfterSort() throws Exception { - SorterTestUtils.testAddAfterSort(ExternalSorter.create(new ExternalSorter.Options()), thrown); + SorterTestUtils.testAddAfterSort(ExternalSorter.create(new ExternalSorter.Options() + .setTempLocation(tmpLocation.toString())), thrown); fail(); } @Test public void testSortTwice() throws Exception { - SorterTestUtils.testSortTwice(ExternalSorter.create(new ExternalSorter.Options()), thrown); + SorterTestUtils.testSortTwice(ExternalSorter.create(new ExternalSorter.Options() + .setTempLocation(tmpLocation.toString())), thrown); fail(); }
