Repository: incubator-beam Updated Branches: refs/heads/master fe00b84dc -> c57643f52
Modify example dependencies to only add runners as optional dependencies. Also support excluding runners by disabling the include-runners profile. Tested ITs BQ and WordCount pass on Dataflow WordCount passes on Flink and Spark Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/cb30ba31 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/cb30ba31 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/cb30ba31 Branch: refs/heads/master Commit: cb30ba310929934fdf4cfdb324fc70f799108742 Parents: fe00b84 Author: Luke Cwik <[email protected]> Authored: Thu Aug 18 16:35:33 2016 -0700 Committer: Luke Cwik <[email protected]> Committed: Fri Aug 19 16:29:43 2016 -0700 ---------------------------------------------------------------------- examples/java/pom.xml | 129 ++++++++++++------- .../org/apache/beam/examples/WordCount.java | 3 +- examples/java8/pom.xml | 40 +++++- 3 files changed, 125 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/cb30ba31/examples/java/pom.xml ---------------------------------------------------------------------- diff --git a/examples/java/pom.xml b/examples/java/pom.xml index 6efbc54..21d7a3a 100644 --- a/examples/java/pom.xml +++ b/examples/java/pom.xml @@ -38,6 +38,78 @@ <spark.version>1.6.2</spark.version> </properties> + <profiles> + <!-- + A default profile that includes optional dependencies + on all of our runners. This is aimed at making it very + easy for users to run examples with any runner without + any configuration. It is not needed or desirable when + testing the examples with a particular runner. + + This profile can be disabled on the command line + by specifying -P !include-runners. + + This profile cannot be lifted to examples-parent because + it would be automatically deactivated when the Java 8 + profile were activated. + --> + <profile> + <id>include-runners</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.beam</groupId> + <artifactId>beam-runners-direct-java</artifactId> + <version>${project.version}</version> + <scope>runtime</scope> + <optional>true</optional> + </dependency> + + <dependency> + <groupId>org.apache.beam</groupId> + <artifactId>beam-runners-flink_2.10</artifactId> + <version>${project.version}</version> + <scope>runtime</scope> + <optional>true</optional> + </dependency> + + <dependency> + <groupId>org.apache.beam</groupId> + <artifactId>beam-runners-google-cloud-dataflow-java</artifactId> + <version>${project.version}</version> + <scope>runtime</scope> + <optional>true</optional> + </dependency> + + <dependency> + <groupId>org.apache.beam</groupId> + <artifactId>beam-runners-spark</artifactId> + <version>${project.version}</version> + <scope>runtime</scope> + <optional>true</optional> + </dependency> + + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-core_2.10</artifactId> + <version>${spark.version}</version> + <scope>runtime</scope> + <optional>true</optional> + </dependency> + + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-streaming_2.10</artifactId> + <version>${spark.version}</version> + <scope>runtime</scope> + <optional>true</optional> + </dependency> + </dependencies> + </profile> + </profiles> + <build> <plugins> <plugin> @@ -270,52 +342,10 @@ </dependency> <dependency> - <groupId>org.apache.beam</groupId> - <artifactId>beam-runners-direct-java</artifactId> - <version>${project.version}</version> - <scope>runtime</scope> - <optional>true</optional> - </dependency> - - <dependency> - <groupId>org.apache.beam</groupId> - <artifactId>beam-runners-google-cloud-dataflow-java</artifactId> - <version>${project.version}</version> - <scope>runtime</scope> - <optional>true</optional> - </dependency> - - <dependency> - <groupId>org.apache.beam</groupId> - <artifactId>beam-runners-flink_2.10</artifactId> - <version>${project.version}</version> - <scope>runtime</scope> - </dependency> - - <dependency> - <groupId>org.apache.beam</groupId> - <artifactId>beam-runners-spark</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-core_2.10</artifactId> - <version>${spark.version}</version> - <scope>runtime</scope> - </dependency> - - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-streaming_2.10</artifactId> - <version>${spark.version}</version> - <scope>runtime</scope> - </dependency> - - <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-jdk14</artifactId> <scope>runtime</scope> + <optional>true</optional> </dependency> <!-- Hamcrest and JUnit are required dependencies of PAssert, @@ -331,6 +361,19 @@ <artifactId>junit</artifactId> </dependency> + <!-- Test dependencies --> + + <!-- + For testing the example itself, use the direct runner. This is separate from + the use of RunnableOnService tests for testing a particular runner. + --> + <dependency> + <groupId>org.apache.beam</groupId> + <artifactId>beam-runners-direct-java</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-all</artifactId> http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/cb30ba31/examples/java/src/main/java/org/apache/beam/examples/WordCount.java ---------------------------------------------------------------------- diff --git a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java index 72ad35f..42d30bb 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java +++ b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java @@ -17,7 +17,6 @@ */ package org.apache.beam.examples; -import org.apache.beam.runners.spark.SparkRunner; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.io.TextIO; import org.apache.beam.sdk.options.Default; @@ -211,7 +210,7 @@ public class WordCount { public static class InputFactory implements DefaultValueFactory<String> { @Override public String create(PipelineOptions options) { - if (options.getRunner().isAssignableFrom(SparkRunner.class)) { + if (options.getRunner().getName().contains("SparkRunner")) { return Resources.getResource("LICENSE").getPath(); } else { return "gs://apache-beam-samples/apache/LICENSE"; http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/cb30ba31/examples/java8/pom.xml ---------------------------------------------------------------------- diff --git a/examples/java8/pom.xml b/examples/java8/pom.xml index 609dcfa..f3f2c3c 100644 --- a/examples/java8/pom.xml +++ b/examples/java8/pom.xml @@ -35,8 +35,11 @@ <packaging>jar</packaging> - <profiles> + <properties> + <spark.version>1.6.2</spark.version> + </properties> + <profiles> <!-- A default profile that includes optional dependencies on all of our runners. This is aimed at making it very @@ -53,7 +56,9 @@ --> <profile> <id>include-runners</id> - <activation><activeByDefault>true</activeByDefault></activation> + <activation> + <activeByDefault>true</activeByDefault> + </activation> <dependencies> <dependency> <groupId>org.apache.beam</groupId> @@ -73,11 +78,35 @@ <dependency> <groupId>org.apache.beam</groupId> + <artifactId>beam-runners-google-cloud-dataflow-java</artifactId> + <version>${project.version}</version> + <scope>runtime</scope> + <optional>true</optional> + </dependency> + + <dependency> + <groupId>org.apache.beam</groupId> <artifactId>beam-runners-spark</artifactId> <version>${project.version}</version> <scope>runtime</scope> <optional>true</optional> </dependency> + + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-core_2.10</artifactId> + <version>${spark.version}</version> + <scope>runtime</scope> + <optional>true</optional> + </dependency> + + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-streaming_2.10</artifactId> + <version>${spark.version}</version> + <scope>runtime</scope> + <optional>true</optional> + </dependency> </dependencies> </profile> </profiles> @@ -165,6 +194,13 @@ </dependency> <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-jdk14</artifactId> + <scope>runtime</scope> + <optional>true</optional> + </dependency> + + <dependency> <groupId>org.apache.avro</groupId> <artifactId>avro</artifactId> </dependency>
