This is an automated email from the ASF dual-hosted git repository. srowen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 1a62e6a [SPARK-36712][BUILD] Make scala-parallel-collections in 2.13 POM a direct dependency (not in maven profile) 1a62e6a is described below commit 1a62e6a2c119df707f15101b03ecff0c3dee62f5 Author: Lukas Rytz <lukas.r...@gmail.com> AuthorDate: Mon Sep 13 11:06:50 2021 -0500 [SPARK-36712][BUILD] Make scala-parallel-collections in 2.13 POM a direct dependency (not in maven profile) As [reported on `devspark.apache.org`](https://lists.apache.org/thread.html/r84cff66217de438f1389899e6d6891b573780159cd45463acf3657aa%40%3Cdev.spark.apache.org%3E), the published POMs when building with Scala 2.13 have the `scala-parallel-collections` dependency only in the `scala-2.13` profile of the pom. ### What changes were proposed in this pull request? This PR suggests to work around this by un-commenting the `scala-parallel-collections` dependency when switching to 2.13 using the the `change-scala-version.sh` script. I included an upgrade to scala-parallel-collections version 1.0.3, the changes compared to 0.2.0 are minor. - removed OSGi metadata - renamed some internal inner classes - added `Automatic-Module-Name` ### Why are the changes needed? According to the posts, this solves issues for developers that write unit tests for their applications. Stephen Coy suggested to use the https://www.mojohaus.org/flatten-maven-plugin. While this sounds like a more principled solution, it is possibly too risky to do at this specific point in time? ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Locally Closes #33948 from lrytz/parCollDep. Authored-by: Lukas Rytz <lukas.r...@gmail.com> Signed-off-by: Sean Owen <sro...@gmail.com> --- core/pom.xml | 15 ++++++--------- dev/change-scala-version.sh | 12 ++++++++---- external/avro/pom.xml | 17 ++++++----------- external/kafka-0-10-sql/pom.xml | 17 ++++++----------- external/kafka-0-10/pom.xml | 18 ++++++------------ mllib/pom.xml | 18 ++++++------------ pom.xml | 16 +++++++--------- sql/catalyst/pom.xml | 18 ++++++------------ sql/core/pom.xml | 15 ++++++--------- sql/hive-thriftserver/pom.xml | 17 ++++++----------- sql/hive/pom.xml | 15 ++++++--------- streaming/pom.xml | 17 ++++++----------- 12 files changed, 75 insertions(+), 120 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index be44964..dbde22f 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -35,6 +35,12 @@ </properties> <dependencies> + <!-- #if scala-2.13 --><!-- + <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> + </dependency> + --><!-- #endif scala-2.13 --> <dependency> <groupId>org.apache.avro</groupId> <artifactId>avro</artifactId> @@ -639,15 +645,6 @@ </plugins> </build> </profile> - <profile> - <id>scala-2.13</id> - <dependencies> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> - </dependency> - </dependencies> - </profile> </profiles> </project> diff --git a/dev/change-scala-version.sh b/dev/change-scala-version.sh index 48b7f64..e17a224 100755 --- a/dev/change-scala-version.sh +++ b/dev/change-scala-version.sh @@ -54,11 +54,15 @@ sed_i() { sed -e "$1" "$2" > "$2.tmp" && mv "$2.tmp" "$2" } -export -f sed_i - BASEDIR=$(dirname $0)/.. -find "$BASEDIR" -name 'pom.xml' -not -path '*target*' -print \ - -exec bash -c "sed_i 's/\(artifactId.*\)_'$FROM_VERSION'/\1_'$TO_VERSION'/g' {}" \; +for f in $(find "$BASEDIR" -name 'pom.xml' -not -path '*target*'); do + echo $f + sed_i 's/\(artifactId.*\)_'$FROM_VERSION'/\1_'$TO_VERSION'/g' $f + sed_i 's/^\([[:space:]]*<!-- #if scala-'$TO_VERSION' -->\)<!--/\1/' $f + sed_i 's/^\([[:space:]]*\)-->\(<!-- #endif scala-'$TO_VERSION' -->\)/\1\2/' $f + sed_i 's/^\([[:space:]]*<!-- #if scala-'$FROM_VERSION' -->\)/\1<!--/' $f + sed_i 's/^\([[:space:]]*\)\(<!-- #endif scala-'$FROM_VERSION' -->\)/\1-->\2/' $f +done # dependency:get is workaround for SPARK-34762 to download the JAR file of commons-cli. # Without this, build with Scala 2.13 using SBT will fail because the help plugin used below downloads only the POM file. diff --git a/external/avro/pom.xml b/external/avro/pom.xml index d9d3583..7e414be 100644 --- a/external/avro/pom.xml +++ b/external/avro/pom.xml @@ -70,22 +70,17 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + <!-- #if scala-2.13 --><!-- + <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> + </dependency> + --><!-- #endif scala-2.13 --> <dependency> <groupId>org.tukaani</groupId> <artifactId>xz</artifactId> </dependency> </dependencies> - <profiles> - <profile> - <id>scala-2.13</id> - <dependencies> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> - </dependency> - </dependencies> - </profile> - </profiles> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory> diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml index 8d505cf..7bedcee 100644 --- a/external/kafka-0-10-sql/pom.xml +++ b/external/kafka-0-10-sql/pom.xml @@ -74,6 +74,12 @@ <type>test-jar</type> <scope>test</scope> </dependency> + <!-- #if scala-2.13 --><!-- + <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> + </dependency> + --><!-- #endif scala-2.13 --> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> @@ -169,17 +175,6 @@ </dependency> </dependencies> - <profiles> - <profile> - <id>scala-2.13</id> - <dependencies> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> - </dependency> - </dependencies> - </profile> - </profiles> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory> diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml index 49f1bef..b689f2b 100644 --- a/external/kafka-0-10/pom.xml +++ b/external/kafka-0-10/pom.xml @@ -59,6 +59,12 @@ <type>test-jar</type> <scope>test</scope> </dependency> + <!-- #if scala-2.13 --><!-- + <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> + </dependency> + --><!-- #endif scala-2.13 --> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> @@ -136,18 +142,6 @@ </dependencies> - <profiles> - <profile> - <id>scala-2.13</id> - <dependencies> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> - </dependency> - </dependencies> - </profile> - </profiles> - <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory> diff --git a/mllib/pom.xml b/mllib/pom.xml index e588961..476226d 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -91,6 +91,12 @@ <type>test-jar</type> <scope>test</scope> </dependency> + <!-- #if scala-2.13 --><!-- + <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> + </dependency> + --><!-- #endif scala-2.13 --> <dependency> <groupId>org.scalanlp</groupId> <artifactId>breeze_${scala.binary.version}</artifactId> @@ -157,18 +163,6 @@ </dependencies> - <profiles> - <profile> - <id>scala-2.13</id> - <dependencies> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> - </dependency> - </dependencies> - </profile> - </profiles> - <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory> diff --git a/pom.xml b/pom.xml index 87a2758..c5cb837 100644 --- a/pom.xml +++ b/pom.xml @@ -399,6 +399,13 @@ <version>${project.version}</version> <type>test-jar</type> </dependency> + <!-- #if scala-2.13 --><!-- + <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> + <version>1.0.3</version> + </dependency> + --><!-- #endif scala-2.13 --> <dependency> <groupId>com.twitter</groupId> <artifactId>chill_${scala.binary.version}</artifactId> @@ -3370,15 +3377,6 @@ <scala.version>2.13.5</scala.version> <scala.binary.version>2.13</scala.binary.version> </properties> - <dependencyManagement> - <dependencies> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> - <version>0.2.0</version> - </dependency> - </dependencies> - </dependencyManagement> <build> <pluginManagement> <plugins> diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index c6612c4..32281b5 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -87,6 +87,12 @@ <artifactId>spark-sketch_${scala.binary.version}</artifactId> <version>${project.version}</version> </dependency> + <!-- #if scala-2.13 --><!-- + <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> + </dependency> + --><!-- #endif scala-2.13 --> <dependency> <groupId>org.scalacheck</groupId> <artifactId>scalacheck_${scala.binary.version}</artifactId> @@ -191,16 +197,4 @@ </plugin> </plugins> </build> - - <profiles> - <profile> - <id>scala-2.13</id> - <dependencies> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> - </dependency> - </dependencies> - </profile> - </profiles> </project> diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 27260ce..f9c12d1 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -90,6 +90,12 @@ <scope>test</scope> </dependency> + <!-- #if scala-2.13 --><!-- + <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> + </dependency> + --><!-- #endif scala-2.13 --> <dependency> <groupId>org.apache.orc</groupId> <artifactId>orc-core</artifactId> @@ -253,15 +259,6 @@ <profiles> <profile> - <id>scala-2.13</id> - <dependencies> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> - </dependency> - </dependencies> - </profile> - <profile> <id>hadoop-2.7</id> <dependencies> <dependency> diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index e14017e..764712f 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -61,6 +61,12 @@ <type>test-jar</type> <scope>test</scope> </dependency> + <!-- #if scala-2.13 --><!-- + <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> + </dependency> + --><!-- #endif scala-2.13 --> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> @@ -138,17 +144,6 @@ <artifactId>commons-cli</artifactId> </dependency> </dependencies> - <profiles> - <profile> - <id>scala-2.13</id> - <dependencies> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> - </dependency> - </dependencies> - </profile> - </profiles> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory> diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 526df64..86cd1ed 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -79,6 +79,12 @@ <type>test-jar</type> <scope>test</scope> </dependency> + <!-- #if scala-2.13 --><!-- + <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> + </dependency> + --><!-- #endif scala-2.13 --> <!-- <dependency> <groupId>com.google.guava</groupId> @@ -217,15 +223,6 @@ </plugins> </build> </profile> - <profile> - <id>scala-2.13</id> - <dependencies> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> - </dependency> - </dependencies> - </profile> </profiles> <build> diff --git a/streaming/pom.xml b/streaming/pom.xml index 9a98aba..ea14151 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -50,6 +50,12 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + <!-- #if scala-2.13 --><!-- + <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> + </dependency> + --><!-- #endif scala-2.13 --> <!-- This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude @@ -132,15 +138,4 @@ </plugins> </build> - <profiles> - <profile> - <id>scala-2.13</id> - <dependencies> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> - </dependency> - </dependencies> - </profile> - </profiles> </project> --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org