This is an automated email from the ASF dual-hosted git repository. aljoscha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink.git
commit f8cbbb505c4ddb4955050689310447e2991a0484 Author: Aljoscha Krettek <[email protected]> AuthorDate: Thu Oct 4 10:44:30 2018 +0200 [FLINK-7811] Add Scala 2.12 profile This changes some modules that had a _2.11 dependency but didn't expose it in their module name to instead depend on the ${scala.binary.version} dependency. The main reason for this is to make the build self contained, before, with the hard-dependency on 2.11, when buildig for 2.12 it would not be clear where the dependency would come from because it is not created as part of the build. This could lead to inconsistencies. For example, when adding a new class in flink-runtime but not recompiling on 2.11 but only on 2.12, the 2.12 tests would fail when using that new class because they would use 2.11 dependencies that weren't rebuilt with the new class. We also don't build flink-scala-shell and flink-connector-kafka-0.8 because they don't work with Scala 2.12. This also includes $PROFILE in dependency convergence check script. This is in preparation for building with "-Dscala-212", where we hace to exclude certain things. We also exclude Kafka 0.8 from stages when building for Scala 2.12 And add Scala 2.12 to the release scripts --- .travis.yml | 14 +-- flink-connectors/flink-connector-kafka-0.9/pom.xml | 28 +++-- flink-connectors/pom.xml | 14 ++- flink-dist/pom.xml | 24 +++-- flink-dist/src/main/assemblies/opt.xml | 4 +- flink-examples/flink-examples-batch/pom.xml | 6 ++ flink-examples/pom.xml | 6 -- flink-formats/flink-avro/pom.xml | 11 +- flink-formats/flink-json/pom.xml | 3 +- flink-formats/flink-parquet/pom.xml | 5 +- flink-libraries/flink-sql-client/pom.xml | 23 ++-- pom.xml | 116 ++++++++++++++++++++- tools/check_dependency_convergence.sh | 14 ++- tools/releasing/create_binary_release.sh | 14 +++ tools/releasing/deploy_staging_jars.sh | 7 +- tools/travis/stage.sh | 12 ++- 16 files changed, 239 insertions(+), 62 deletions(-) diff --git a/.travis.yml b/.travis.yml index d4b8aba..e2291f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,25 +76,25 @@ jobs: # main profile - stage: compile script: ./tools/travis_controller.sh - env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws" + env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" name: compile - stage: test script: ./tools/travis_controller.sh - env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws" + env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" name: core - script: ./tools/travis_controller.sh - env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws" + env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" name: libraries - script: ./tools/travis_controller.sh - env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws" + env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" name: connectors - script: ./tools/travis_controller.sh - env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws" + env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" name: tests - script: ./tools/travis_controller.sh - env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws" + env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" name: misc - stage: cleanup script: ./tools/travis_controller.sh - env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws" + env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" name: cleanup diff --git a/flink-connectors/flink-connector-kafka-0.9/pom.xml b/flink-connectors/flink-connector-kafka-0.9/pom.xml index 3e0aa46..9b9bbef 100644 --- a/flink-connectors/flink-connector-kafka-0.9/pom.xml +++ b/flink-connectors/flink-connector-kafka-0.9/pom.xml @@ -136,14 +136,6 @@ under the License. </dependency> <dependency> - <!-- include 0.9 server for tests --> - <groupId>org.apache.kafka</groupId> - <artifactId>kafka_${scala.binary.version}</artifactId> - <version>${kafka.version}</version> - <scope>test</scope> - </dependency> - - <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-metrics-jmx</artifactId> <version>${project.version}</version> @@ -191,6 +183,26 @@ under the License. </dependencies> <profiles> + <!-- The kafka_${...} dependency doesn't exist for Scala 2.12, so we need to exclude + this (test-)dependency when building for Scala 2.12. --> + <profile> + <id>scala-2.11</id> + <activation> + <property> + <name>!scala-2.12</name> + </property> + </activation> + <dependencies> + <dependency> + <!-- include 0.9 server for tests --> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka_${scala.binary.version}</artifactId> + <version>${kafka.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + </profile> + <!-- Create SQL Client uber jars by default --> <profile> <id>sql-jars</id> diff --git a/flink-connectors/pom.xml b/flink-connectors/pom.xml index d7a851a..78aab46 100644 --- a/flink-connectors/pom.xml +++ b/flink-connectors/pom.xml @@ -42,7 +42,6 @@ under the License. <module>flink-hbase</module> <module>flink-hcatalog</module> <module>flink-connector-kafka-base</module> - <module>flink-connector-kafka-0.8</module> <module>flink-connector-kafka-0.9</module> <module>flink-connector-kafka-0.10</module> <module>flink-connector-kafka-0.11</module> @@ -83,6 +82,19 @@ under the License. <!-- See main pom.xml for explanation of profiles --> <profiles> + <!-- there's no Kafka 0.8 dependency for Scala 2.12, we only include when building + for Scala 2.11 --> + <profile> + <id>scala-2.11</id> + <activation> + <property> + <name>!scala-2.12</name> + </property> + </activation> + <modules> + <module>flink-connector-kafka-0.8</module> + </modules> + </profile> <!-- We include the kinesis module only optionally because it contains a dependency licenced under the "Amazon Software License". diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml index d857a30..0ed0ff5 100644 --- a/flink-dist/pom.xml +++ b/flink-dist/pom.xml @@ -100,12 +100,6 @@ under the License. <dependency> <groupId>org.apache.flink</groupId> - <artifactId>flink-scala-shell_${scala.binary.version}</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>org.apache.flink</groupId> <artifactId>flink-metrics-core</artifactId> <version>${project.version}</version> </dependency> @@ -375,6 +369,24 @@ under the License. <profiles> <profile> + <id>scala-2.11</id> + <activation> + <property> + <name>!scala-2.12</name> + </property> + </activation> + <!-- Scala Shell doesn't currently work with Scala 2.12 so only include + when building for Scala 2.11. --> + <dependencies> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-scala-shell_${scala.binary.version}</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + </profile> + + <profile> <!-- Copies that shaded Hadoop uber jar to the dist folder. --> <id>include-hadoop</id> <activation> diff --git a/flink-dist/src/main/assemblies/opt.xml b/flink-dist/src/main/assemblies/opt.xml index 136f0f3..6fe661a 100644 --- a/flink-dist/src/main/assemblies/opt.xml +++ b/flink-dist/src/main/assemblies/opt.xml @@ -69,9 +69,9 @@ <!-- SQL Client --> <file> - <source>../flink-libraries/flink-sql-client/target/flink-sql-client-${project.version}.jar</source> + <source>../flink-libraries/flink-sql-client/target/flink-sql-client_${scala.binary.version}-${project.version}.jar</source> <outputDirectory>opt/</outputDirectory> - <destName>flink-sql-client-${project.version}.jar</destName> + <destName>flink-sql-client_${scala.binary.version}-${project.version}.jar</destName> <fileMode>0644</fileMode> </file> diff --git a/flink-examples/flink-examples-batch/pom.xml b/flink-examples/flink-examples-batch/pom.xml index 8e0d9e0..24ef016 100644 --- a/flink-examples/flink-examples-batch/pom.xml +++ b/flink-examples/flink-examples-batch/pom.xml @@ -44,6 +44,12 @@ under the License. <artifactId>flink-scala_${scala.binary.version}</artifactId> <version>${project.version}</version> </dependency> + + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-clients_${scala.binary.version}</artifactId> + <version>${project.version}</version> + </dependency> </dependencies> diff --git a/flink-examples/pom.xml b/flink-examples/pom.xml index 3b8d17a..6a9b7e4 100644 --- a/flink-examples/pom.xml +++ b/flink-examples/pom.xml @@ -48,12 +48,6 @@ under the License. <version>${project.version}</version> </dependency> - <dependency> - <groupId>org.apache.flink</groupId> - <artifactId>flink-clients_${scala.binary.version}</artifactId> - <version>${project.version}</version> - </dependency> - <!-- Add a logging Framework, to make the examples produce --> <!-- logs when executing in the IDE --> diff --git a/flink-formats/flink-avro/pom.xml b/flink-formats/flink-avro/pom.xml index 421c91e..867d947 100644 --- a/flink-formats/flink-avro/pom.xml +++ b/flink-formats/flink-avro/pom.xml @@ -82,7 +82,7 @@ under the License. <!-- TODO This could be dropped if we move the Table Avro IT Case --> <dependency> <groupId>org.apache.flink</groupId> - <artifactId>flink-table_2.11</artifactId> + <artifactId>flink-table_${scala.binary.version}</artifactId> <version>${project.version}</version> <scope>test</scope> </dependency> @@ -90,25 +90,22 @@ under the License. <!-- TODO This could be dropped if we move the Table Avro IT Case --> <dependency> <groupId>org.apache.flink</groupId> - <artifactId>flink-table_2.11</artifactId> + <artifactId>flink-table_${scala.binary.version}</artifactId> <version>${project.version}</version> <scope>test</scope> <type>test-jar</type> </dependency> - <!-- To avoid having to have the 'flink-avro' project dependent on a particular - Scala version, we hard-refer the flink-streaming-scala_2.11 here --> - <dependency> <groupId>org.apache.flink</groupId> - <artifactId>flink-streaming-scala_2.11</artifactId> + <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId> <version>${project.version}</version> <scope>test</scope> </dependency> <dependency> <groupId>org.apache.flink</groupId> - <artifactId>flink-test-utils_2.11</artifactId> + <artifactId>flink-test-utils_${scala.binary.version}</artifactId> <version>${project.version}</version> <scope>test</scope> </dependency> diff --git a/flink-formats/flink-json/pom.xml b/flink-formats/flink-json/pom.xml index 4b1baa9..1d64289 100644 --- a/flink-formats/flink-json/pom.xml +++ b/flink-formats/flink-json/pom.xml @@ -77,8 +77,7 @@ under the License. <!-- TODO This could be dropped if we change JsonRowFormatFactoryTest --> <dependency> <groupId>org.apache.flink</groupId> - <!-- use a dedicated Scala version to not depend on it --> - <artifactId>flink-table_2.11</artifactId> + <artifactId>flink-table_${scala.binary.version}</artifactId> <version>${project.version}</version> <scope>test</scope> </dependency> diff --git a/flink-formats/flink-parquet/pom.xml b/flink-formats/flink-parquet/pom.xml index e338d05..b195801 100644 --- a/flink-formats/flink-parquet/pom.xml +++ b/flink-formats/flink-parquet/pom.xml @@ -96,14 +96,14 @@ under the License. <dependency> <groupId>org.apache.flink</groupId> - <artifactId>flink-test-utils_2.11</artifactId> + <artifactId>flink-test-utils_${scala.binary.version}</artifactId> <version>${project.version}</version> <scope>test</scope> </dependency> <dependency> <groupId>org.apache.flink</groupId> - <artifactId>flink-streaming-java_2.11</artifactId> + <artifactId>flink-streaming-java_${scala.binary.version}</artifactId> <version>${project.version}</version> <scope>test</scope> </dependency> @@ -117,7 +117,6 @@ under the License. </dependencies> - <build> <plugins> <!-- Generate Test class from avro schema --> diff --git a/flink-libraries/flink-sql-client/pom.xml b/flink-libraries/flink-sql-client/pom.xml index b7b21f8..c0299b8 100644 --- a/flink-libraries/flink-sql-client/pom.xml +++ b/flink-libraries/flink-sql-client/pom.xml @@ -27,7 +27,7 @@ under the License. <relativePath>..</relativePath> </parent> - <artifactId>flink-sql-client</artifactId> + <artifactId>flink-sql-client_${scala.binary.version}</artifactId> <name>flink-sql-client</name> <packaging>jar</packaging> @@ -45,28 +45,25 @@ under the License. <dependency> <groupId>org.apache.flink</groupId> - <!-- use a dedicated Scala version to not depend on it --> - <artifactId>flink-clients_2.11</artifactId> + <artifactId>flink-clients_${scala.binary.version}</artifactId> <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.flink</groupId> - <!-- use a dedicated Scala version to not depend on it --> - <artifactId>flink-streaming-scala_2.11</artifactId> + <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId> <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.flink</groupId> - <!-- use a dedicated Scala version to not depend on it --> - <artifactId>flink-table_2.11</artifactId> + <artifactId>flink-table_${scala.binary.version}</artifactId> <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.flink</groupId> - <artifactId>flink-cep_2.11</artifactId> + <artifactId>flink-cep_${scala.binary.version}</artifactId> <version>${project.version}</version> </dependency> @@ -113,8 +110,7 @@ under the License. <dependency> <groupId>org.apache.flink</groupId> - <!-- use a dedicated Scala version to not depend on it --> - <artifactId>flink-runtime_2.11</artifactId> + <artifactId>flink-runtime_${scala.binary.version}</artifactId> <version>${project.version}</version> <type>test-jar</type> <scope>test</scope> @@ -122,16 +118,14 @@ under the License. <dependency> <groupId>org.apache.flink</groupId> - <!-- use a dedicated Scala version to not depend on it --> - <artifactId>flink-test-utils_2.11</artifactId> + <artifactId>flink-test-utils_${scala.binary.version}</artifactId> <version>${project.version}</version> <scope>test</scope> </dependency> <dependency> <groupId>org.apache.flink</groupId> - <!-- use a dedicated Scala version to not depend on it --> - <artifactId>flink-clients_2.11</artifactId> + <artifactId>flink-clients_${scala.binary.version}</artifactId> <version>${project.version}</version> <type>test-jar</type> <scope>test</scope> @@ -141,6 +135,7 @@ under the License. <build> <plugins> + <!-- Build flink-sql-client jar --> <plugin> <groupId>org.apache.maven.plugins</groupId> diff --git a/pom.xml b/pom.xml index 4b75c10..2a5dfce 100644 --- a/pom.xml +++ b/pom.xml @@ -75,7 +75,6 @@ under the License. <module>flink-test-utils-parent</module> <module>flink-state-backends</module> <module>flink-libraries</module> - <module>flink-scala-shell</module> <module>flink-quickstart</module> <module>flink-contrib</module> <module>flink-dist</module> @@ -112,7 +111,8 @@ under the License. <maven.compiler.source>${java.version}</maven.compiler.source> <maven.compiler.target>${java.version}</maven.compiler.target> <scala.macros.version>2.1.0</scala.macros.version> - <!-- Default scala versions, may be overwritten by build profiles --> + <!-- Default scala versions, must be overwritten by build profiles, so we set something + invalid here --> <scala.version>2.11.12</scala.version> <scala.binary.version>2.11</scala.binary.version> <chill.version>0.7.6</chill.version> @@ -620,6 +620,118 @@ under the License. <profiles> <profile> + <id>scala-2.11</id> + <properties> + <scala.version>2.11.12</scala.version> + <scala.binary.version>2.11</scala.binary.version> + </properties> + <activation> + <property> + <name>!scala-2.12</name> + </property> + </activation> + <!-- Scala Shell doesn't currently work with Scala 2.12 so only include + when building for Scala 2.11. --> + <modules> + <module>flink-scala-shell</module> + </modules> + <build> + <plugins> + <!-- make sure we don't have any _2.10 or _2.12 dependencies when building + for Scala 2.11 --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <executions> + <execution> + <id>enforce-versions</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <bannedDependencies> + <excludes combine.children="append"> + <exclude>*:*_2.12</exclude> + <exclude>*:*_2.10</exclude> + </excludes> + </bannedDependencies> + </rules> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + + + <profile> + <id>scala-2.12</id> + <properties> + <scala.version>2.12.7</scala.version> + <scala.binary.version>2.12</scala.binary.version> + </properties> + <activation> + <property> + <name>scala-2.12</name> + </property> + </activation> + <build> + <plugins> + <!-- don't run tests that don't work for Scala 2.12, because not all of the + required test dependencies are available for Scala 2.12. The Kafka 0.9 connector + still works with Scala 2.12 because it only needs the scala-version-independent + kafka-clients dependency at runtime. --> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.12</version> + <executions> + <execution> + <id>regex-property</id> + <goals> + <goal>regex-property</goal> + </goals> + <configuration> + <name>maven.test.skip</name> + <value>${project.artifactId}</value> + <regex>(flink-connector-kafka-0.9.*)|(flink-scala-shell.*)</regex> + <replacement>true</replacement> + <failIfNoMatch>false</failIfNoMatch> + </configuration> + </execution> + </executions> + </plugin> + <!-- make sure we don't have any _2.10 or _2.11 dependencies when building + for Scala 2.12 --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <executions> + <execution> + <id>enforce-versions</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <bannedDependencies> + <excludes combine.children="append"> + <exclude>*:*_2.11</exclude> + <exclude>*:*_2.10</exclude> + </excludes> + </bannedDependencies> + </rules> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + + <profile> <id>java9</id> <activation> <jdk>9</jdk> diff --git a/tools/check_dependency_convergence.sh b/tools/check_dependency_convergence.sh index 5db8d9d..a9294b8 100755 --- a/tools/check_dependency_convergence.sh +++ b/tools/check_dependency_convergence.sh @@ -43,6 +43,18 @@ modules=$(find . -maxdepth 3 -name 'pom.xml' -printf '%h\n' | sort -u | grep "fl for module in ${modules} do + # There are no Scala 2.12 dependencies for older Kafka versions + if [[ $PROFILE == *"scala-2.12"* ]]; then + if [[ $module == *"kafka-0.8"* ]]; then + echo "excluding ${module} because we build for Scala 2.12" + continue 2 + fi + if [[ $module == *"kafka-0.9"* ]]; then + echo "excluding ${module} because we build for Scala 2.12" + continue 2 + fi + fi + # we are only interested in child modules for other_module in ${modules} do @@ -54,7 +66,7 @@ do cd "${module}" echo "checking ${module}" - output=$(mvn validate -nsu -Dcheckstyle.skip=true -Dcheck-convergence) + output=$(mvn validate $PROFILE -nsu -Dcheckstyle.skip=true -Dcheck-convergence) exit_code=$? if [[ ${exit_code} != 0 ]]; then echo "dependency convergence failed." diff --git a/tools/releasing/create_binary_release.sh b/tools/releasing/create_binary_release.sh index 4dbc6fa..8146127 100755 --- a/tools/releasing/create_binary_release.sh +++ b/tools/releasing/create_binary_release.sh @@ -69,7 +69,16 @@ make_binary_release() { dir_name="flink-$RELEASE_VERSION-bin-$NAME-scala_${SCALA_VERSION}" fi + if [ $SCALA_VERSION = "2.12" ]; then + FLAGS="$FLAGS -Dscala-2.12" + elif [ $SCALA_VERSION = "2.11" ]; then + FLAGS="$FLAGS -Dscala-2.11" + else + echo "Invalid Scala version ${SCALA_VERSION}" + fi + # enable release profile here (to check for the maven version) + tools/change-scala-version.sh ${SCALA_VERSION} $MVN clean package $FLAGS -Prelease -pl flink-shaded-hadoop/flink-shaded-hadoop2-uber,flink-dist -am -Dgpg.skip -Dcheckstyle.skip=true -DskipTests -Dmaven.test.skip=true cd flink-dist/target/flink-*-bin/ @@ -88,6 +97,11 @@ make_binary_release() { } if [ "$SCALA_VERSION" == "none" ] && [ "$HADOOP_VERSION" == "none" ]; then + make_binary_release "" "-DwithoutHadoop" "2.12" + make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "2.12" + make_binary_release "hadoop26" "-Dhadoop.version=2.6.5" "2.12" + make_binary_release "hadoop27" "-Dhadoop.version=2.7.5" "2.12" + make_binary_release "hadoop28" "-Dhadoop.version=2.8.3" "2.12" make_binary_release "" "-DwithoutHadoop" "2.11" make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "2.11" make_binary_release "hadoop26" "-Dhadoop.version=2.6.5" "2.11" diff --git a/tools/releasing/deploy_staging_jars.sh b/tools/releasing/deploy_staging_jars.sh index 1bc20d0..7fd08cf 100755 --- a/tools/releasing/deploy_staging_jars.sh +++ b/tools/releasing/deploy_staging_jars.sh @@ -40,6 +40,11 @@ cd .. echo "Deploying to repository.apache.org" +COMMON_OPTIONS="-Prelease,docs-and-source -DskipTests -DretryFailedDeploymentCount=10" + echo "Deploying Scala 2.11 version" -$MVN clean deploy -Prelease,docs-and-source -DskipTests -DretryFailedDeploymentCount=10 +$MVN clean deploy $COMMON_OPTIONS -Dscala-2.11 + +echo "Deploying Scala 2.12 version" +$MVN clean deploy $COMMON_OPTIONS -Dscala-2.12 diff --git a/tools/travis/stage.sh b/tools/travis/stage.sh index 3b2252e..53f6bee 100644 --- a/tools/travis/stage.sh +++ b/tools/travis/stage.sh @@ -35,7 +35,6 @@ flink-optimizer,\ flink-runtime,\ flink-runtime-web,\ flink-scala,\ -flink-scala-shell,\ flink-streaming-java,\ flink-streaming-scala" @@ -74,7 +73,6 @@ flink-connectors/flink-connector-elasticsearch5,\ flink-connectors/flink-connector-elasticsearch6,\ flink-connectors/flink-connector-elasticsearch-base,\ flink-connectors/flink-connector-filesystem,\ -flink-connectors/flink-connector-kafka-0.8,\ flink-connectors/flink-connector-kafka-0.9,\ flink-connectors/flink-connector-kafka-0.10,\ flink-connectors/flink-connector-kafka-0.11,\ @@ -90,6 +88,16 @@ if [[ ${PROFILE} == *"include-kinesis"* ]]; then MODULES_CONNECTORS="$MODULES_CONNECTORS,flink-connectors/flink-connector-kinesis" fi +# we can only build the Kafka 0.8 connector when building for Scala 2.11 +if [[ $PROFILE == *"scala-2.11"* ]]; then + MODULES_CONNECTORS="$MODULES_CONNECTORS,flink-connectors/flink-connector-kafka-0.8" +fi + +# we can only build the Scala Shell when building for Scala 2.11 +if [[ $PROFILE == *"scala-2.11"* ]]; then + MODULES_CORE="$MODULES_CORE,flink-scala-shell" +fi + function get_compile_modules_for_stage() { local stage=$1
