Repository: beam Updated Branches: refs/heads/master 0e429b33f -> 11010cf08
Make modules that depend on Hadoop and Spark use the same version property Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ce4e5174 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ce4e5174 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ce4e5174 Branch: refs/heads/master Commit: ce4e51747501111ae2c4b1691c6994bd0f92e161 Parents: 0e429b3 Author: Ismaël MejÃa <[email protected]> Authored: Sun Jun 4 22:55:05 2017 +0200 Committer: Jean-Baptiste Onofré <[email protected]> Committed: Mon Jul 3 08:19:02 2017 +0200 ---------------------------------------------------------------------- examples/java/pom.xml | 18 ++++-- examples/java8/pom.xml | 18 ++++-- pom.xml | 65 +++++++++++++++++++- runners/apex/pom.xml | 2 +- runners/spark/pom.xml | 7 --- sdks/java/extensions/sorter/pom.xml | 6 -- sdks/java/io/hadoop-file-system/pom.xml | 31 ---------- sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 2 - sdks/java/io/hbase/pom.xml | 9 ++- sdks/java/io/hcatalog/pom.xml | 6 +- sdks/java/io/jdbc/pom.xml | 2 - sdks/java/io/pom.xml | 31 ---------- sdks/java/javadoc/pom.xml | 2 - .../main/resources/archetype-resources/pom.xml | 1 - .../main/resources/archetype-resources/pom.xml | 1 - 15 files changed, 98 insertions(+), 103 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/ce4e5174/examples/java/pom.xml ---------------------------------------------------------------------- diff --git a/examples/java/pom.xml b/examples/java/pom.xml index 701e4fe..7ae4e6a 100644 --- a/examples/java/pom.xml +++ b/examples/java/pom.xml @@ -34,10 +34,6 @@ <packaging>jar</packaging> - <properties> - <spark.version>1.6.2</spark.version> - </properties> - <profiles> <!-- @@ -66,6 +62,12 @@ <groupId>org.apache.beam</groupId> <artifactId>beam-runners-apex</artifactId> <scope>runtime</scope> + <exclusions> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + </exclusions> </dependency> <!-- Apex depends on httpclient version 4.3.5, project has a transitive dependency to httpclient 4.0.1 from @@ -95,6 +97,12 @@ <groupId>org.apache.beam</groupId> <artifactId>beam-runners-flink_2.10</artifactId> <scope>runtime</scope> + <exclusions> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + </exclusions> </dependency> </dependencies> </profile> @@ -116,13 +124,11 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-streaming_2.10</artifactId> - <version>${spark.version}</version> <scope>runtime</scope> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_2.10</artifactId> - <version>${spark.version}</version> <scope>runtime</scope> <exclusions> <exclusion> http://git-wip-us.apache.org/repos/asf/beam/blob/ce4e5174/examples/java8/pom.xml ---------------------------------------------------------------------- diff --git a/examples/java8/pom.xml b/examples/java8/pom.xml index 56295a4..a0ce708 100644 --- a/examples/java8/pom.xml +++ b/examples/java8/pom.xml @@ -35,10 +35,6 @@ <packaging>jar</packaging> - <properties> - <spark.version>1.6.2</spark.version> - </properties> - <profiles> <!-- The direct runner is available by default. @@ -66,6 +62,12 @@ <groupId>org.apache.beam</groupId> <artifactId>beam-runners-apex</artifactId> <scope>runtime</scope> + <exclusions> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + </exclusions> </dependency> <!-- Apex depends on httpclient version 4.3.5, project has a transitive dependency to httpclient 4.0.1 from @@ -95,6 +97,12 @@ <groupId>org.apache.beam</groupId> <artifactId>beam-runners-flink_2.10</artifactId> <scope>runtime</scope> + <exclusions> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + </exclusions> </dependency> </dependencies> </profile> @@ -116,13 +124,11 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-streaming_2.10</artifactId> - <version>${spark.version}</version> <scope>runtime</scope> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_2.10</artifactId> - <version>${spark.version}</version> <scope>runtime</scope> <exclusions> <exclusion> http://git-wip-us.apache.org/repos/asf/beam/blob/ce4e5174/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index fe51660..c0207ef 100644 --- a/pom.xml +++ b/pom.xml @@ -127,6 +127,12 @@ <guava.version>20.0</guava.version> <grpc.version>1.2.0</grpc.version> <grpc-google-common-protos.version>0.1.9</grpc-google-common-protos.version> + <!-- + This is the version of Hadoop used to compile the module that depend on Hadoop. + This dependency is defined with a provided scope. + Users must supply their own Hadoop version at runtime. + --> + <hadoop.version>2.7.3</hadoop.version> <hamcrest.version>1.3</hamcrest.version> <jackson.version>2.8.9</jackson.version> <findbugs.version>3.0.1</findbugs.version> @@ -139,7 +145,7 @@ <pubsub.version>v1-rev10-1.22.0</pubsub.version> <slf4j.version>1.7.14</slf4j.version> <spanner.version>0.20.0-beta</spanner.version> - <spark.version>1.6.2</spark.version> + <spark.version>1.6.3</spark.version> <spring.version>4.3.5.RELEASE</spring.version> <stax2.version>3.1.4</stax2.version> <storage.version>v1-rev71-1.22.0</storage.version> @@ -1075,6 +1081,42 @@ <version>${snappy-java.version}</version> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${hadoop.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-core_2.10</artifactId> + <version>${spark.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-streaming_2.10</artifactId> + <version>${spark.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-network-common_2.10</artifactId> + <version>${spark.version}</version> + </dependency> + <!-- Testing --> <dependency> @@ -1144,6 +1186,27 @@ <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <version>${hadoop.version}</version> + <classifier>tests</classifier> + <scope>test</scope> + </dependency> </dependencies> </dependencyManagement> http://git-wip-us.apache.org/repos/asf/beam/blob/ce4e5174/runners/apex/pom.xml ---------------------------------------------------------------------- diff --git a/runners/apex/pom.xml b/runners/apex/pom.xml index 2c54654..88ff0f2 100644 --- a/runners/apex/pom.xml +++ b/runners/apex/pom.xml @@ -261,7 +261,7 @@ <ignoredUsedUndeclaredDependency>com.esotericsoftware.kryo:kryo::${apex.kryo.version}</ignoredUsedUndeclaredDependency> <ignoredUsedUndeclaredDependency>com.datatorrent:netlet::1.3.0</ignoredUsedUndeclaredDependency> <ignoredUsedUndeclaredDependency>org.slf4j:slf4j-api:jar:1.7.14</ignoredUsedUndeclaredDependency> - <ignoredUsedUndeclaredDependency>org.apache.hadoop:hadoop-common:jar:2.6.0</ignoredUsedUndeclaredDependency> + <ignoredUsedUndeclaredDependency>org.apache.hadoop:hadoop-common:jar:${hadoop.version}</ignoredUsedUndeclaredDependency> <ignoredUsedUndeclaredDependency>joda-time:joda-time:jar:2.4</ignoredUsedUndeclaredDependency> <ignoredUsedUndeclaredDependency>com.google.guava:guava:jar:20.0</ignoredUsedUndeclaredDependency> </ignoredUsedUndeclaredDependencies> http://git-wip-us.apache.org/repos/asf/beam/blob/ce4e5174/runners/spark/pom.xml ---------------------------------------------------------------------- diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml index ee72dd9..1d93427 100644 --- a/runners/spark/pom.xml +++ b/runners/spark/pom.xml @@ -34,8 +34,6 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> - <spark.version>1.6.3</spark.version> - <hadoop.version>2.2.0</hadoop.version> <kafka.version>0.9.0.1</kafka.version> <jackson.version>2.4.4</jackson.version> <dropwizard.metrics.version>3.1.2</dropwizard.metrics.version> @@ -135,31 +133,26 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_2.10</artifactId> - <version>${spark.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-streaming_2.10</artifactId> - <version>${spark.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-network-common_2.10</artifactId> - <version>${spark.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> - <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/beam/blob/ce4e5174/sdks/java/extensions/sorter/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/sorter/pom.xml b/sdks/java/extensions/sorter/pom.xml index 9d25f9d..ac61f76 100644 --- a/sdks/java/extensions/sorter/pom.xml +++ b/sdks/java/extensions/sorter/pom.xml @@ -29,10 +29,6 @@ <artifactId>beam-sdks-java-extensions-sorter</artifactId> <name>Apache Beam :: SDKs :: Java :: Extensions :: Sorter</name> - <properties> - <hadoop.version>2.7.1</hadoop.version> - </properties> - <dependencies> <dependency> <groupId>org.apache.beam</groupId> @@ -42,14 +38,12 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> - <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - <version>${hadoop.version}</version> <scope>provided</scope> </dependency> http://git-wip-us.apache.org/repos/asf/beam/blob/ce4e5174/sdks/java/io/hadoop-file-system/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/io/hadoop-file-system/pom.xml b/sdks/java/io/hadoop-file-system/pom.xml index db5a1db..a54977e 100644 --- a/sdks/java/io/hadoop-file-system/pom.xml +++ b/sdks/java/io/hadoop-file-system/pom.xml @@ -44,37 +44,6 @@ </plugins> </build> - <properties> - <!-- - This is the version of Hadoop used to compile the hadoop-common module. - This dependency is defined with a provided scope. - Users must supply their own Hadoop version at runtime. - --> - <hadoop.version>2.7.3</hadoop.version> - </properties> - - <dependencyManagement> - <!-- - We define dependencies here instead of sdks/java/io because - of a version mimatch between this Hadoop version and other - Hadoop versions declared in other io submodules. - --> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdfs</artifactId> - <classifier>tests</classifier> - <version>${hadoop.version}</version> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-minicluster</artifactId> - <version>${hadoop.version}</version> - </dependency> - </dependencies> - </dependencyManagement> - <dependencies> <dependency> <groupId>org.apache.beam</groupId> http://git-wip-us.apache.org/repos/asf/beam/blob/ce4e5174/sdks/java/io/hadoop/jdk1.8-tests/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/io/hadoop/jdk1.8-tests/pom.xml b/sdks/java/io/hadoop/jdk1.8-tests/pom.xml index 9f84e88..baaa982 100644 --- a/sdks/java/io/hadoop/jdk1.8-tests/pom.xml +++ b/sdks/java/io/hadoop/jdk1.8-tests/pom.xml @@ -108,13 +108,11 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-streaming_2.10</artifactId> - <version>${spark.version}</version> <scope>runtime</scope> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_2.10</artifactId> - <version>${spark.version}</version> <scope>runtime</scope> <exclusions> <exclusion> http://git-wip-us.apache.org/repos/asf/beam/blob/ce4e5174/sdks/java/io/hbase/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/io/hbase/pom.xml b/sdks/java/io/hbase/pom.xml index 4d9d600..9d5e2aa 100644 --- a/sdks/java/io/hbase/pom.xml +++ b/sdks/java/io/hbase/pom.xml @@ -32,7 +32,6 @@ <properties> <hbase.version>1.2.6</hbase.version> - <hbase.hadoop.version>2.5.1</hbase.hadoop.version> </properties> <build> @@ -109,14 +108,18 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minicluster</artifactId> - <version>${hbase.hadoop.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> <scope>test</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - <version>${hbase.hadoop.version}</version> <scope>test</scope> </dependency> http://git-wip-us.apache.org/repos/asf/beam/blob/ce4e5174/sdks/java/io/hcatalog/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/io/hcatalog/pom.xml b/sdks/java/io/hcatalog/pom.xml index 19b62a5..8af740d 100644 --- a/sdks/java/io/hcatalog/pom.xml +++ b/sdks/java/io/hcatalog/pom.xml @@ -39,14 +39,14 @@ <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> + <artifactId>maven-shade-plugin</artifactId> <configuration> - <redirectTestOutputToFile>true</redirectTestOutputToFile> + <createDependencyReducedPom>false</createDependencyReducedPom> </configuration> </plugin> </plugins> </build> - + <dependencies> <dependency> <groupId>org.apache.beam</groupId> http://git-wip-us.apache.org/repos/asf/beam/blob/ce4e5174/sdks/java/io/jdbc/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/io/jdbc/pom.xml b/sdks/java/io/jdbc/pom.xml index 17c26a0..45ec06c 100644 --- a/sdks/java/io/jdbc/pom.xml +++ b/sdks/java/io/jdbc/pom.xml @@ -49,13 +49,11 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-streaming_2.10</artifactId> - <version>${spark.version}</version> <scope>runtime</scope> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_2.10</artifactId> - <version>${spark.version}</version> <scope>runtime</scope> <exclusions> <exclusion> http://git-wip-us.apache.org/repos/asf/beam/blob/ce4e5174/sdks/java/io/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/io/pom.xml b/sdks/java/io/pom.xml index e5db41b..458dfaf 100644 --- a/sdks/java/io/pom.xml +++ b/sdks/java/io/pom.xml @@ -32,37 +32,6 @@ <description>Beam SDK Java IO provides different connectivity components (sources and sinks) to consume and produce data from systems.</description> - <properties> - <!-- - This is the version of Hadoop used to compile the hadoop-common module. - This dependency is defined with a provided scope. - Users must supply their own Hadoop version at runtime. - --> - <hadoop.version>2.7.3</hadoop.version> - </properties> - - <dependencyManagement> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <version>${hadoop.version}</version> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <version>${hadoop.version}</version> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-core</artifactId> - <version>${hadoop.version}</version> - </dependency> - </dependencies> - </dependencyManagement> - <modules> <module>amqp</module> <module>cassandra</module> http://git-wip-us.apache.org/repos/asf/beam/blob/ce4e5174/sdks/java/javadoc/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/javadoc/pom.xml b/sdks/java/javadoc/pom.xml index 54dae3a..08d5ec6 100644 --- a/sdks/java/javadoc/pom.xml +++ b/sdks/java/javadoc/pom.xml @@ -196,13 +196,11 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_2.10</artifactId> - <version>${spark.version}</version> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-streaming_2.10</artifactId> - <version>${spark.version}</version> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/beam/blob/ce4e5174/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml index af4fbd3..4517861 100644 --- a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml +++ b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml @@ -242,7 +242,6 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-streaming_2.10</artifactId> - <version>${spark.version}</version> <scope>runtime</scope> <exclusions> <exclusion> http://git-wip-us.apache.org/repos/asf/beam/blob/ce4e5174/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml ---------------------------------------------------------------------- diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml index b8b9c9f..d039ddb 100644 --- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml +++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml @@ -241,7 +241,6 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-streaming_2.10</artifactId> - <version>${spark.version}</version> <scope>runtime</scope> <exclusions> <exclusion>
