Repository: beam Updated Branches: refs/heads/master 0a6211b56 -> 178381992
[BEAM-1310] Add running integration tests in JdbcIO on Spark and Dataflow runners Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/eb87ffc7 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/eb87ffc7 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/eb87ffc7 Branch: refs/heads/master Commit: eb87ffc769df6ab410b90a599489034a83aede16 Parents: 0a6211b Author: Stephen Sisk <[email protected]> Authored: Thu Feb 23 14:01:18 2017 -0800 Committer: Jean-Baptiste Onofré <[email protected]> Committed: Fri Mar 3 08:05:16 2017 +0100 ---------------------------------------------------------------------- sdks/java/io/jdbc/pom.xml | 50 ++++++++++++++++++++ .../org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 27 ++++++----- sdks/java/io/pom.xml | 37 +++++++++++++++ 3 files changed, 102 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/eb87ffc7/sdks/java/io/jdbc/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/io/jdbc/pom.xml b/sdks/java/io/jdbc/pom.xml index afe428a..fd5c52b 100644 --- a/sdks/java/io/jdbc/pom.xml +++ b/sdks/java/io/jdbc/pom.xml @@ -30,6 +30,56 @@ <name>Apache Beam :: SDKs :: Java :: IO :: JDBC</name> <description>IO to read and write on JDBC datasource.</description> + <!-- + The dataflow-runner and spark-runner profiles support using those runners during an integration test. These are + not the long-term way we want to support using runners in ITs (e.g. it is annoying to add to all IO modules.) + We cannot create a dependency IO -> Runners since the runners depend on IO (e.g. kafka depends on spark.) + --> + + <profiles> + <!-- Include the Apache Spark runner -P spark-runner --> + <profile> + <id>spark-runner</id> + <dependencies> + <dependency> + <groupId>org.apache.beam</groupId> + <artifactId>beam-runners-spark</artifactId> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-streaming_2.10</artifactId> + <version>${spark.version}</version> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-core_2.10</artifactId> + <version>${spark.version}</version> + <scope>runtime</scope> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>jul-to-slf4j</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> + </profile> + + <!-- Include the Google Cloud Dataflow runner -P dataflow-runner --> + <profile> + <id>dataflow-runner</id> + <dependencies> + <dependency> + <groupId>org.apache.beam</groupId> + <artifactId>beam-runners-google-cloud-dataflow-java</artifactId> + <scope>runtime</scope> + </dependency> + </dependencies> + </profile> + </profiles> + <dependencies> <dependency> <groupId>org.apache.beam</groupId> http://git-wip-us.apache.org/repos/asf/beam/blob/eb87ffc7/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java index 15206c7..a09db48 100644 --- a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java +++ b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java @@ -38,6 +38,7 @@ import org.apache.beam.sdk.values.PCollection; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -50,16 +51,19 @@ import org.postgresql.ds.PGSimpleDataSource; * <p>This test requires a running instance of Postgres, and the test dataset must exist in the * database. `JdbcTestDataSet` will create the read table. * - * <p>You can run just this test by doing the following: + * <p>You can run this test by doing the following: * <pre> - * mvn test-compile compile failsafe:integration-test -D beamTestPipelineOptions='[ - * "--postgresServerName=1.2.3.4", - * "--postgresUsername=postgres", - * "--postgresDatabaseName=myfancydb", - * "--postgresPassword=yourpassword", - * "--postgresSsl=false" - * ]' -DskipITs=false -Dit.test=org.apache.beam.sdk.io.jdbc.JdbcIOIT -DfailIfNoTests=false + * mvn -e -Pio-it verify -pl sdks/java/io/jdbc -DintegrationTestPipelineOptions='[ + * "--postgresServerName=1.2.3.4", + * "--postgresUsername=postgres", + * "--postgresDatabaseName=myfancydb", + * "--postgresPassword=mypass", + * "--postgresSsl=false" ]' * </pre> + * + * <p>If you want to run this with a runner besides directrunner, there are profiles for dataflow + * and spark in the jdbc pom. You'll want to activate those in addition to the normal test runner + * invocation pipeline options. */ @RunWith(JUnit4.class) public class JdbcIOIT { @@ -103,6 +107,9 @@ public class JdbcIOIT { } } + @Rule + public TestPipeline pipeline = TestPipeline.create(); + /** * Does a test read of a few rows from a postgres database. * @@ -113,8 +120,6 @@ public class JdbcIOIT { public void testRead() throws SQLException { String tableName = JdbcTestDataSet.READ_TABLE_NAME; - TestPipeline pipeline = TestPipeline.create(); - PCollection<KV<String, Integer>> output = pipeline.apply(JdbcIO.<KV<String, Integer>>read() .withDataSourceConfiguration(JdbcIO.DataSourceConfiguration.create(dataSource)) .withQuery("select name,id from " + tableName) @@ -148,8 +153,6 @@ public class JdbcIOIT { public void testWrite() throws SQLException { writeTableName = JdbcTestDataSet.createWriteDataTable(dataSource); - TestPipeline pipeline = TestPipeline.create(); - ArrayList<KV<Integer, String>> data = new ArrayList<>(); for (int i = 0; i < 1000; i++) { KV<Integer, String> kv = KV.of(i, "Test"); http://git-wip-us.apache.org/repos/asf/beam/blob/eb87ffc7/sdks/java/io/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/io/pom.xml b/sdks/java/io/pom.xml index 082f26b..8d5b69b 100644 --- a/sdks/java/io/pom.xml +++ b/sdks/java/io/pom.xml @@ -77,4 +77,41 @@ <module>mqtt</module> </modules> + <profiles> + <profile> + <id>io-it</id> + <activation> + <property><name>io-it</name></property> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>integration-test</goal> + <goal>verify</goal> + </goals> + <configuration> + <useManifestOnlyJar>false</useManifestOnlyJar> + <skip>${skipDefaultIT}</skip> + <parallel>all</parallel> + <threadCount>4</threadCount> + <systemPropertyVariables> + <beamTestPipelineOptions>${integrationTestPipelineOptions}</beamTestPipelineOptions> + </systemPropertyVariables> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + <properties> + <skipITs>false</skipITs> + </properties> + </profile> + </profiles> + </project>
