Repository: incubator-beam Updated Branches: refs/heads/master 1c5858b86 -> c5744ccf2
Migrated IO display data tests must be runner-filesystem agnostic Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/1117a033 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/1117a033 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/1117a033 Branch: refs/heads/master Commit: 1117a0331485d654ceb0d6ce424f681d48fab5ea Parents: 1c5858b Author: Scott Wegner <[email protected]> Authored: Fri Jul 8 15:53:43 2016 -0700 Committer: Dan Halperin <[email protected]> Committed: Mon Jul 11 12:55:18 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/beam/sdk/io/AvroIOTest.java | 20 +++++++++++++++++--- .../org/apache/beam/sdk/io/BigQueryIOTest.java | 14 ++++++++++---- .../java/org/apache/beam/sdk/io/TextIOTest.java | 20 +++++++++++++++++--- 3 files changed, 44 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/1117a033/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java index 047e7d0..026724a 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java @@ -29,10 +29,12 @@ import static org.junit.Assert.assertTrue; import org.apache.beam.sdk.coders.AvroCoder; import org.apache.beam.sdk.coders.DefaultCoder; import org.apache.beam.sdk.io.AvroIO.Write.Bound; +import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.testing.NeedsRunner; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.RunnableOnService; import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.testing.TestPipelineOptions; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator; @@ -47,6 +49,8 @@ import org.apache.avro.Schema; import org.apache.avro.file.DataFileReader; import org.apache.avro.generic.GenericRecord; import org.apache.avro.reflect.Nullable; +import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -69,6 +73,11 @@ public class AvroIOTest { @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); + @BeforeClass + public static void setupClass() { + IOChannelUtils.registerStandardIOFactories(TestPipeline.testingPipelineOptions()); + } + @Test public void testReadWithoutValidationFlag() throws Exception { AvroIO.Read.Bound<GenericRecord> read = AvroIO.Read.from("gs://bucket/foo*/baz"); @@ -313,11 +322,16 @@ public class AvroIOTest { @Test @Category(RunnableOnService.class) - public void testPrimitiveWriteDisplayData() { - DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(); + @Ignore("[BEAM-436] DirectRunner RunnableOnService tempLocation configuration insufficient") + public void testPrimitiveWriteDisplayData() throws IOException { + PipelineOptions options = DisplayDataEvaluator.getDefaultOptions(); + String tempRoot = options.as(TestPipelineOptions.class).getTempRoot(); + String outputPath = IOChannelUtils.getFactory(tempRoot).resolve(tempRoot, "foo"); + + DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(options); AvroIO.Write.Bound<?> write = AvroIO.Write - .to("foo") + .to(outputPath) .withSchema(Schema.create(Schema.Type.STRING)) .withoutValidation(); http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/1117a033/sdks/java/core/src/test/java/org/apache/beam/sdk/io/BigQueryIOTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/BigQueryIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/BigQueryIOTest.java index 78d950e..0d1a9f8 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/BigQueryIOTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/BigQueryIOTest.java @@ -94,6 +94,7 @@ import org.hamcrest.CoreMatchers; import org.hamcrest.Matchers; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -643,8 +644,9 @@ public class BigQueryIOTest implements Serializable { @Test @Category(RunnableOnService.class) + @Ignore("[BEAM-436] DirectRunner RunnableOnService tempLocation configuration insufficient") public void testTableSourcePrimitiveDisplayData() throws IOException, InterruptedException { - DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(bqOptions); + DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(); BigQueryIO.Read.Bound read = BigQueryIO.Read .from("project:dataset.tableId") .withTestServices(new FakeBigQueryServices() @@ -659,8 +661,9 @@ public class BigQueryIOTest implements Serializable { @Test @Category(RunnableOnService.class) + @Ignore("[BEAM-436] DirectRunner RunnableOnService tempLocation configuration insufficient") public void testQuerySourcePrimitiveDisplayData() throws IOException, InterruptedException { - DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(bqOptions); + DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(); BigQueryIO.Read.Bound read = BigQueryIO.Read .fromQuery("foobar") .withTestServices(new FakeBigQueryServices() @@ -684,20 +687,23 @@ public class BigQueryIOTest implements Serializable { @Test @Category(RunnableOnService.class) + @Ignore("[BEAM-436] DirectRunner RunnableOnService tempLocation configuration insufficient") public void testBatchSinkPrimitiveDisplayData() throws IOException, InterruptedException { testSinkPrimitiveDisplayData(/* streaming: */ false); } @Test @Category(RunnableOnService.class) + @Ignore("[BEAM-436] DirectRunner RunnableOnService tempLocation configuration insufficient") public void testStreamingSinkPrimitiveDisplayData() throws IOException, InterruptedException { testSinkPrimitiveDisplayData(/* streaming: */ true); } private void testSinkPrimitiveDisplayData(boolean streaming) throws IOException, InterruptedException { - bqOptions.as(StreamingOptions.class).setStreaming(streaming); - DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(bqOptions); + PipelineOptions options = TestPipeline.testingPipelineOptions(); + options.as(StreamingOptions.class).setStreaming(streaming); + DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(options); BigQueryIO.Write.Bound write = BigQueryIO.Write .to("project:dataset.table") http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/1117a033/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java index 28e9ea4..babb50a 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java @@ -42,12 +42,14 @@ import org.apache.beam.sdk.io.BoundedSource.BoundedReader; import org.apache.beam.sdk.io.TextIO.CompressionType; import org.apache.beam.sdk.io.TextIO.TextSource; import org.apache.beam.sdk.options.GcsOptions; +import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.testing.NeedsRunner; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.RunnableOnService; import org.apache.beam.sdk.testing.SourceTestUtils; import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.testing.TestPipelineOptions; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.transforms.display.DisplayData; import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator; @@ -59,6 +61,8 @@ import org.apache.beam.sdk.values.PCollection; import com.google.common.collect.ImmutableList; +import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -101,6 +105,11 @@ public class TextIOTest { @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); @Rule public ExpectedException expectedException = ExpectedException.none(); + @BeforeClass + public static void setupClass() { + IOChannelUtils.registerStandardIOFactories(TestPipeline.testingPipelineOptions()); + } + <T> void runTestRead(T[] expected, Coder<T> coder) throws Exception { File tmpFile = tmpFolder.newFile("file.txt"); String filename = tmpFile.getPath(); @@ -336,14 +345,19 @@ public class TextIOTest { @Test @Category(RunnableOnService.class) - public void testPrimitiveWriteDisplayData() { + @Ignore("[BEAM-436] DirectRunner RunnableOnService tempLocation configuration insufficient") + public void testPrimitiveWriteDisplayData() throws IOException { + PipelineOptions options = DisplayDataEvaluator.getDefaultOptions(); + String tempRoot = options.as(TestPipelineOptions.class).getTempRoot(); + String outputPath = IOChannelUtils.getFactory(tempRoot).resolve(tempRoot, "foobar"); + DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(); - TextIO.Write.Bound<?> write = TextIO.Write.to("foobar"); + TextIO.Write.Bound<?> write = TextIO.Write.to(outputPath); Set<DisplayData> displayData = evaluator.displayDataForPrimitiveTransforms(write); assertThat("TextIO.Write should include the file prefix in its primitive display data", - displayData, hasItem(hasDisplayItem(hasValue(startsWith("foobar"))))); + displayData, hasItem(hasDisplayItem(hasValue(startsWith(outputPath))))); } @Test
