Repository: flink Updated Branches: refs/heads/master 7582390c1 -> 5242e9668
[FLINK-1637] Reduce number of files in uberjar for java 6 This closes #450 Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/5242e966 Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/5242e966 Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/5242e966 Branch: refs/heads/master Commit: 5242e9668a54cb6e74ee4f33718d35cf856b0703 Parents: 7582390 Author: Robert Metzger <rmetz...@apache.org> Authored: Tue Mar 3 12:44:21 2015 +0100 Committer: Robert Metzger <rmetz...@apache.org> Committed: Mon Mar 9 09:46:51 2015 +0100 ---------------------------------------------------------------------- .travis.yml | 8 +++---- flink-dist/pom.xml | 23 -------------------- .../examples/java/wordcount/WordCount.java | 3 ++- tools/create_release_files.sh | 5 +++++ tools/deploy_to_maven.sh | 9 ++++---- tools/travis_mvn_watchdog.sh | 19 ++++++++++++++++ 6 files changed, 35 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/5242e966/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 48d4913..3c4e10d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,13 +14,13 @@ language: java #See https://issues.apache.org/jira/browse/FLINK-1072 matrix: include: - - jdk: "openjdk6" - env: PROFILE="-Dhadoop.profile=1" - jdk: "oraclejdk7" env: PROFILE="-Dhadoop.profile=1" - - jdk: "openjdk6" + - jdk: "openjdk6" # this will also deploy a uberjar to s3 at some point + env: PROFILE="-Dhadoop.profile=1" + - jdk: "openjdk7" env: PROFILE="-P!include-yarn -Dhadoop.version=2.0.0-alpha" - - jdk: "oraclejdk7" + - jdk: "openjdk6" # we must use openjdk6 here to deploy a java6 compatible uber-jar for YARN env: PROFILE="-Dhadoop.version=2.2.0" - jdk: "oraclejdk8" env: PROFILE="-Dhadoop.version=2.6.0" http://git-wip-us.apache.org/repos/asf/flink/blob/5242e966/flink-dist/pom.xml ---------------------------------------------------------------------- diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml index 1ac8bea..af3b100 100644 --- a/flink-dist/pom.xml +++ b/flink-dist/pom.xml @@ -115,12 +115,6 @@ under the License. <dependency> <groupId>org.apache.flink</groupId> - <artifactId>flink-streaming-connectors</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>org.apache.flink</groupId> <artifactId>flink-streaming-examples</artifactId> <version>${project.version}</version> </dependency> @@ -236,23 +230,6 @@ under the License. </profile> <profile> - <id>hadoop-2</id> - <activation> - <property> - <!-- Please do not remove the 'hadoop2' comment. See ./tools/generate_specific_pom.sh --> - <!--hadoop2--><name>!hadoop.profile</name> - </property> - </activation> - <dependencies> - <dependency> - <groupId>org.apache.flink</groupId> - <artifactId>flink-hbase</artifactId> - <version>${project.version}</version> - </dependency> - </dependencies> - </profile> - - <profile> <id>debian-package</id> <build> <plugins> http://git-wip-us.apache.org/repos/asf/flink/blob/5242e966/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/WordCount.java ---------------------------------------------------------------------- diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/WordCount.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/WordCount.java index 6be5368..07aa2ce 100644 --- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/WordCount.java +++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/WordCount.java @@ -53,7 +53,8 @@ public class WordCount { // ************************************************************************* public static void main(String[] args) throws Exception { - + + if(!parseParameters(args)) { return; } http://git-wip-us.apache.org/repos/asf/flink/blob/5242e966/tools/create_release_files.sh ---------------------------------------------------------------------- diff --git a/tools/create_release_files.sh b/tools/create_release_files.sh index 09b4e7a..c9b1aaa 100755 --- a/tools/create_release_files.sh +++ b/tools/create_release_files.sh @@ -28,6 +28,11 @@ # # https://github.com/apache/spark/blob/master/dev/create-release/create-release.sh # + +# Added by rmetzger +echo "Please make sure to use java 6 OPENJDK (not oracle) to build the release binaries" +exit 1 + CURR_DIR=`pwd` if [[ `basename $CURR_DIR` != "tools" ]] ; then echo "You have to call the script from the tools/ dir" http://git-wip-us.apache.org/repos/asf/flink/blob/5242e966/tools/deploy_to_maven.sh ---------------------------------------------------------------------- diff --git a/tools/deploy_to_maven.sh b/tools/deploy_to_maven.sh index f4290af..1781927 100755 --- a/tools/deploy_to_maven.sh +++ b/tools/deploy_to_maven.sh @@ -27,9 +27,7 @@ # 4. deploy snapshot and s3 (hadoop2 - 2.2.0) # 5. Nothing (hadoop2 - 2.5.1) -# Changes (since travis changed the id assignment) -# switched 2. with 3. -# switched 5. with 6. + echo "install lifecylce mapping fake plugin" git clone https://github.com/mfriedenhagen/dummy-lifecycle-mapping-plugin.git @@ -47,9 +45,11 @@ function getVersion() { fi flink_home="`dirname \"$here\"`" cd $flink_home - echo `mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version|grep -Ev '(^\[|Download\w+:)'` + echo `mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -E '^([0-9]+.[0-9]+(.[0-9]+)?(-[a-zA-Z0-9]+)?)$'` } +pwd + # this will take a while CURRENT_FLINK_VERSION=`getVersion` if [[ "$CURRENT_FLINK_VERSION" == *-SNAPSHOT ]]; then @@ -60,6 +60,7 @@ fi echo "detected current version as: '$CURRENT_FLINK_VERSION' ; hadoop1: $CURRENT_FLINK_VERSION_HADOOP1 " + # Check if push/commit is eligible for pushing echo "Job: $TRAVIS_JOB_NUMBER ; isPR: $TRAVIS_PULL_REQUEST" if [[ $TRAVIS_PULL_REQUEST == "false" ]] ; then http://git-wip-us.apache.org/repos/asf/flink/blob/5242e966/tools/travis_mvn_watchdog.sh ---------------------------------------------------------------------- diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh index e770a97..d94f977 100755 --- a/tools/travis_mvn_watchdog.sh +++ b/tools/travis_mvn_watchdog.sh @@ -179,5 +179,24 @@ rm $MVN_EXIT upload_artifacts_s3 +# Check the number of files in the uber jar and fail the build if there are too many files (see: FLINK-1637) + +# since we are in flink/tools/artifacts +# we are going back to +cd ../../ + + +UBERJAR=`find . | grep uberjar | head -n 1` +if [ -z "$UBERJAR" ] ; then + echo "Uberjar not found. Assuming failed build"; +else + jar tf $UBERJAR | wc -l > num_files_in_uberjar + NUM_FILES_IN_UBERJAR=`cat num_files_in_uberjar` + echo "Files in uberjar: $NUM_FILES_IN_UBERJAR. Uberjar: $UBERJAR" + if [ "$NUM_FILES_IN_UBERJAR" -ge "65536" ] ; then + echo "WARN: The number of files in the uberjar ($NUM_FILES_IN_UBERJAR) exceeds the maximum number of possible files for Java 6 (65536)" + fi +fi + # Exit code for Travis build success/failure exit $EXIT_CODE
