Repository: flink Updated Branches: refs/heads/master 99c627892 -> 2438824a8
[FLINK-6466] [build] Build Hadoop 2.8.0 convenience binaries Update Hadoop versions and replace Hadoop 2.3 with 2.8 in build and continuous integration scripts. flink-yarn-tests can now be enabled for all supported Hadoop versions and is made non-optional. This closes #3832 Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2438824a Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2438824a Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2438824a Branch: refs/heads/master Commit: 2438824a8f39c07783e80adf902c72eecab31347 Parents: 99c6278 Author: Greg Hogan <[email protected]> Authored: Fri May 5 08:39:53 2017 -0400 Committer: Greg Hogan <[email protected]> Committed: Thu Jun 1 12:06:58 2017 -0400 ---------------------------------------------------------------------- .travis.yml | 24 +++---- .../flink-shaded-hadoop2/pom.xml | 6 +- .../flink-shaded-include-yarn-tests/pom.xml | 72 -------------------- .../flink-shaded-yarn-tests/pom.xml | 72 ++++++++++++++++++++ flink-shaded-hadoop/pom.xml | 10 +-- flink-yarn-tests/pom.xml | 2 +- pom.xml | 11 +-- tools/create_release_files.sh | 32 ++++----- 8 files changed, 107 insertions(+), 122 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/2438824a/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 44ea97d..b9251b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,32 +16,32 @@ matrix: include: # Always run test groups A and B together - jdk: "oraclejdk8" - env: PROFILE="-Dhadoop.version=2.7.2 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-a,include-kinesis,jdk8 -Dmaven.javadoc.skip=true" + env: PROFILE="-Dhadoop.version=2.8.0 -Dscala-2.11 -Pflink-fast-tests-a,include-kinesis,jdk8 -Dmaven.javadoc.skip=true" - jdk: "oraclejdk8" - env: PROFILE="-Dhadoop.version=2.7.2 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-b,include-kinesis,jdk8 -Dmaven.javadoc.skip=true" + env: PROFILE="-Dhadoop.version=2.8.0 -Dscala-2.11 -Pflink-fast-tests-b,include-kinesis,jdk8 -Dmaven.javadoc.skip=true" - jdk: "oraclejdk8" - env: PROFILE="-Dhadoop.version=2.7.2 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-c,include-kinesis,jdk8 -Dmaven.javadoc.skip=true" + env: PROFILE="-Dhadoop.version=2.8.0 -Dscala-2.11 -Pflink-fast-tests-c,include-kinesis,jdk8 -Dmaven.javadoc.skip=true" - jdk: "oraclejdk8" - env: PROFILE="-Dhadoop.version=2.6.3 -Pinclude-yarn-tests,flink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true" + env: PROFILE="-Dhadoop.version=2.7.3 -Pflink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true" - jdk: "oraclejdk8" - env: PROFILE="-Dhadoop.version=2.6.3 -Pinclude-yarn-tests,flink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true" + env: PROFILE="-Dhadoop.version=2.7.3 -Pflink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true" - jdk: "oraclejdk8" - env: PROFILE="-Dhadoop.version=2.6.3 -Pinclude-yarn-tests,flink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true" + env: PROFILE="-Dhadoop.version=2.7.3 -Pflink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true" - jdk: "openjdk7" - env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true" + env: PROFILE="-Dhadoop.version=2.6.5 -Dscala-2.11 -Pflink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true" - jdk: "openjdk7" - env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true" + env: PROFILE="-Dhadoop.version=2.6.5 -Dscala-2.11 -Pflink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true" - jdk: "openjdk7" - env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true" + env: PROFILE="-Dhadoop.version=2.6.5 -Dscala-2.11 -Pflink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true" - jdk: "oraclejdk7" - env: PROFILE="-Dhadoop.version=2.3.0 -Pflink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true" + env: PROFILE="-Dhadoop.version=2.4.1 -Pflink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true" - jdk: "oraclejdk7" - env: PROFILE="-Dhadoop.version=2.3.0 -Pflink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true" + env: PROFILE="-Dhadoop.version=2.4.1 -Pflink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true" - jdk: "oraclejdk7" - env: PROFILE="-Dhadoop.version=2.3.0 -Pflink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true" + env: PROFILE="-Dhadoop.version=2.4.1 -Pflink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true" git: http://git-wip-us.apache.org/repos/asf/flink/blob/2438824a/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml ---------------------------------------------------------------------- diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml b/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml index a1159d6..74754c3 100644 --- a/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml +++ b/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml @@ -681,13 +681,13 @@ under the License. <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpcore</artifactId> - <version>4.2.5</version> + <version>4.4.4</version> </dependency> - + <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> - <version>4.2.6</version> + <version>4.5.2</version> </dependency> </dependencies> </dependencyManagement> http://git-wip-us.apache.org/repos/asf/flink/blob/2438824a/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml ---------------------------------------------------------------------- diff --git a/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml b/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml deleted file mode 100644 index fe87331..0000000 --- a/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml +++ /dev/null @@ -1,72 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - - <modelVersion>4.0.0</modelVersion> - - <parent> - <groupId>org.apache.flink</groupId> - <artifactId>flink-shaded-hadoop</artifactId> - <version>1.4-SNAPSHOT</version> - <relativePath>..</relativePath> - </parent> - - <artifactId>flink-shaded-include-yarn-tests</artifactId> - <name>flink-shaded-include-yarn-tests</name> - - <packaging>jar</packaging> - - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <version>${hadoop.version}</version> - </dependency> - <!-- NOTE: We do not exclude hadoop's dependencies so that the tests start properly (in particular jetty) --> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-server-tests</artifactId> - <version>${hadoop.version}</version> - </dependency> - - <!-- Add test-jar as well --> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-server-tests</artifactId> - <version>${hadoop.version}</version> - <type>test-jar</type> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-minicluster</artifactId> - <version>${hadoop.version}</version> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-server-resourcemanager</artifactId> - <version>${hadoop.version}</version> - </dependency> - - </dependencies> - -</project> http://git-wip-us.apache.org/repos/asf/flink/blob/2438824a/flink-shaded-hadoop/flink-shaded-yarn-tests/pom.xml ---------------------------------------------------------------------- diff --git a/flink-shaded-hadoop/flink-shaded-yarn-tests/pom.xml b/flink-shaded-hadoop/flink-shaded-yarn-tests/pom.xml new file mode 100644 index 0000000..e3ad97d --- /dev/null +++ b/flink-shaded-hadoop/flink-shaded-yarn-tests/pom.xml @@ -0,0 +1,72 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.flink</groupId> + <artifactId>flink-shaded-hadoop</artifactId> + <version>1.4-SNAPSHOT</version> + <relativePath>..</relativePath> + </parent> + + <artifactId>flink-shaded-yarn-tests</artifactId> + <name>flink-shaded-yarn-tests</name> + + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + </dependency> + <!-- NOTE: We do not exclude hadoop's dependencies so that the tests start properly (in particular jetty) --> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-tests</artifactId> + <version>${hadoop.version}</version> + </dependency> + + <!-- Add test-jar as well --> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-tests</artifactId> + <version>${hadoop.version}</version> + <type>test-jar</type> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <version>${hadoop.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-resourcemanager</artifactId> + <version>${hadoop.version}</version> + </dependency> + + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/flink/blob/2438824a/flink-shaded-hadoop/pom.xml ---------------------------------------------------------------------- diff --git a/flink-shaded-hadoop/pom.xml b/flink-shaded-hadoop/pom.xml index a549ae9..26e6077 100644 --- a/flink-shaded-hadoop/pom.xml +++ b/flink-shaded-hadoop/pom.xml @@ -37,6 +37,7 @@ under the License. <modules> <module>flink-shaded-hadoop2</module> <module>flink-shaded-hadoop2-uber</module> + <module>flink-shaded-yarn-tests</module> </modules> <dependencies> @@ -63,15 +64,6 @@ under the License. </dependency> </dependencies> - <profiles> - <profile> - <id>include-yarn-tests</id> - <modules> - <module>flink-shaded-include-yarn-tests</module> - </modules> - </profile> - </profiles> - <build> <plugins> <!-- Relocate the Hadoop's Guava dependency into a different namespace and http://git-wip-us.apache.org/repos/asf/flink/blob/2438824a/flink-yarn-tests/pom.xml ---------------------------------------------------------------------- diff --git a/flink-yarn-tests/pom.xml b/flink-yarn-tests/pom.xml index 200e5ab..f198ece 100644 --- a/flink-yarn-tests/pom.xml +++ b/flink-yarn-tests/pom.xml @@ -88,7 +88,7 @@ under the License. <dependency> <groupId>org.apache.flink</groupId> - <artifactId>flink-shaded-include-yarn-tests</artifactId> + <artifactId>flink-shaded-yarn-tests</artifactId> <version>${project.version}</version> <exclusions> <exclusion> http://git-wip-us.apache.org/repos/asf/flink/blob/2438824a/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index ae92be9..ac020b6 100644 --- a/pom.xml +++ b/pom.xml @@ -76,6 +76,7 @@ under the License. <module>flink-mesos</module> <module>flink-metrics</module> <module>flink-yarn</module> + <module>flink-yarn-tests</module> <module>flink-fs-tests</module> </modules> @@ -84,7 +85,7 @@ under the License. <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <!-- Internal property to reduce build times on TravisCi --> <flink-fast-tests-pattern>never-match-me</flink-fast-tests-pattern> - <hadoop.version>2.3.0</hadoop.version> + <hadoop.version>2.4.1</hadoop.version> <!-- Need to use a user property here because the surefire forkCount is not exposed as a property. With this we can set it on the "mvn" commandline in travis. --> @@ -493,14 +494,6 @@ under the License. </properties> </profile> - <!-- Profile to deactivate the YARN tests --> - <profile> - <id>include-yarn-tests</id> - <modules> - <module>flink-yarn-tests</module> - </modules> - </profile> - <profile> <id>vendor-repos</id> <!-- Add vendor maven repositories --> http://git-wip-us.apache.org/repos/asf/flink/blob/2438824a/tools/create_release_files.sh ---------------------------------------------------------------------- diff --git a/tools/create_release_files.sh b/tools/create_release_files.sh index 40d038c..ba99846 100755 --- a/tools/create_release_files.sh +++ b/tools/create_release_files.sh @@ -91,7 +91,7 @@ fi usage() { set +x - echo "./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.2" + echo "./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.3" echo "" echo "usage:" echo "[--scala-version <version>] [--hadoop-version <version>]" @@ -102,12 +102,12 @@ usage() { echo " USER_NAME=APACHEID GPG_PASSPHRASE=XXX GPG_KEY=KEYID \ " echo " GIT_AUTHOR=\"`git config --get user.name` <`git config --get user.email`>\" \ " echo " GIT_REPO=github.com/apache/flink.git \ " - echo " ./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.2" + echo " ./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.3" echo "" echo "example 2: build local release" echo " NEW_VERSION=1.2.0 RELEASE_BRANCH=master OLD_VERSION=1.2-SNAPSHOT \ " echo " GPG_PASSPHRASE=XXX GPG_KEY=XXX IS_LOCAL_DIST=true \ " - echo " ./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.2" + echo " ./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.3" exit 1 } @@ -272,25 +272,25 @@ make_source_release # build dist by input parameter of "--scala-vervion xxx --hadoop-version xxx" if [ "$SCALA_VERSION" == "none" ] && [ "$HADOOP_VERSION" == "none" ]; then - make_binary_release "hadoop2" "" 2.10 - make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "2.10" - make_binary_release "hadoop26" "-Dhadoop.version=2.6.3" "2.10" - make_binary_release "hadoop27" "-Dhadoop.version=2.7.2" "2.10" - - make_binary_release "hadoop2" "" 2.11 - make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "2.11" - make_binary_release "hadoop26" "-Dhadoop.version=2.6.3" "2.11" - make_binary_release "hadoop27" "-Dhadoop.version=2.7.2" "2.11" + make_binary_release "hadoop2" "" "2.10" + make_binary_release "hadoop26" "-Dhadoop.version=2.6.5" "2.10" + make_binary_release "hadoop27" "-Dhadoop.version=2.7.3" "2.10" + make_binary_release "hadoop28" "-Dhadoop.version=2.8.0" "2.10" + + make_binary_release "hadoop2" "" "2.11" + make_binary_release "hadoop26" "-Dhadoop.version=2.6.5" "2.11" + make_binary_release "hadoop27" "-Dhadoop.version=2.7.3" "2.11" + make_binary_release "hadoop28" "-Dhadoop.version=2.8.0" "2.11" elif [ "$SCALA_VERSION" == none ] && [ "$HADOOP_VERSION" != "none" ] then make_binary_release "hadoop2" "-Dhadoop.version=$HADOOP_VERSION" "2.10" make_binary_release "hadoop2" "-Dhadoop.version=$HADOOP_VERSION" "2.11" elif [ "$SCALA_VERSION" != none ] && [ "$HADOOP_VERSION" == "none" ] then - make_binary_release "hadoop2" "" $SCALA_VERSION - make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "$SCALA_VERSION" - make_binary_release "hadoop26" "-Dhadoop.version=2.6.3" "$SCALA_VERSION" - make_binary_release "hadoop27" "-Dhadoop.version=2.7.2" "$SCALA_VERSION" + make_binary_release "hadoop2" "" "$SCALA_VERSION" + make_binary_release "hadoop26" "-Dhadoop.version=2.6.5" "$SCALA_VERSION" + make_binary_release "hadoop27" "-Dhadoop.version=2.7.3" "$SCALA_VERSION" + make_binary_release "hadoop28" "-Dhadoop.version=2.8.0" "$SCALA_VERSION" else make_binary_release "hadoop2x" "-Dhadoop.version=$HADOOP_VERSION" "$SCALA_VERSION" fi
