This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 498b7e1a2f3 [SPARK-45272][BUILD][INFRA][DOCS] Remove Scala version
specific comments, and scala-2.13 profile usage
498b7e1a2f3 is described below
commit 498b7e1a2f3691ded5bd0772e77f79cee71134b5
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Sep 22 00:05:32 2023 -0700
[SPARK-45272][BUILD][INFRA][DOCS] Remove Scala version specific comments,
and scala-2.13 profile usage
### What changes were proposed in this pull request?
This PR is a followup of https://github.com/apache/spark/pull/43008 that
cleanups Scala version specific comments, and `scala-2.13` profile usage.
See also https://github.com/apache/spark/pull/43008#discussion_r1333866152
### Why are the changes needed?
To remove unnecessary profile specifications
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
CI in this PR should test them out. For README.md, I manually checked via
GitHub.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #43049 from HyukjinKwon/SPARK-44113-followup-2.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
connector/avro/pom.xml | 2 --
connector/connect/server/pom.xml | 2 --
connector/kafka-0-10-sql/pom.xml | 2 --
connector/kafka-0-10/pom.xml | 2 --
connector/protobuf/pom.xml | 2 --
core/pom.xml | 2 --
dev/mima | 8 ++++----
docs/_plugins/copy_api_dirs.rb | 12 ++++++------
docs/building-spark.md | 2 ++
mllib/pom.xml | 2 --
pom.xml | 12 ++++++------
sql/catalyst/pom.xml | 2 --
sql/core/pom.xml | 2 --
sql/hive-thriftserver/pom.xml | 2 --
sql/hive/pom.xml | 2 --
streaming/pom.xml | 2 --
16 files changed, 18 insertions(+), 40 deletions(-)
diff --git a/connector/avro/pom.xml b/connector/avro/pom.xml
index bde5d5de6ea..412486b3fca 100644
--- a/connector/avro/pom.xml
+++ b/connector/avro/pom.xml
@@ -70,12 +70,10 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
- <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- <!-- #endif scala-2.13 -->
<dependency>
<groupId>org.tukaani</groupId>
<artifactId>xz</artifactId>
diff --git a/connector/connect/server/pom.xml b/connector/connect/server/pom.xml
index 291da8d8706..e9c7bd86e0f 100644
--- a/connector/connect/server/pom.xml
+++ b/connector/connect/server/pom.xml
@@ -152,12 +152,10 @@
</exclusion>
</exclusions>
</dependency>
- <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- <!-- #endif scala-2.13 -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
diff --git a/connector/kafka-0-10-sql/pom.xml b/connector/kafka-0-10-sql/pom.xml
index 56f0b874b2b..41e573c8c41 100644
--- a/connector/kafka-0-10-sql/pom.xml
+++ b/connector/kafka-0-10-sql/pom.xml
@@ -74,12 +74,10 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- <!-- #endif scala-2.13 -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
diff --git a/connector/kafka-0-10/pom.xml b/connector/kafka-0-10/pom.xml
index 4c76c8a9fe4..29a2ecd2491 100644
--- a/connector/kafka-0-10/pom.xml
+++ b/connector/kafka-0-10/pom.xml
@@ -59,12 +59,10 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- <!-- #endif scala-2.13 -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
diff --git a/connector/protobuf/pom.xml b/connector/protobuf/pom.xml
index 7854a9c59dc..8c52576c353 100644
--- a/connector/protobuf/pom.xml
+++ b/connector/protobuf/pom.xml
@@ -70,12 +70,10 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
- <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- <!-- #endif scala-2.13 -->
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
diff --git a/core/pom.xml b/core/pom.xml
index 6e5e3132042..bbfac34117b 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -35,12 +35,10 @@
</properties>
<dependencies>
- <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- <!-- #endif scala-2.13 -->
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
diff --git a/dev/mima b/dev/mima
index 17e9abe4bfb..859301b4d66 100755
--- a/dev/mima
+++ b/dev/mima
@@ -24,9 +24,9 @@ set -e
FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
cd "$FWDIR"
-SPARK_PROFILES=${1:-"-Pscala-2.13 -Pmesos -Pkubernetes -Pyarn
-Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"}
-TOOLS_CLASSPATH="$(build/sbt -Pscala-2.13 -DcopyDependencies=false "export
tools/fullClasspath" | grep jar | tail -n1)"
-OLD_DEPS_CLASSPATH="$(build/sbt -Pscala-2.13 -DcopyDependencies=false
$SPARK_PROFILES "export oldDeps/fullClasspath" | grep jar | tail -n1)"
+SPARK_PROFILES=${1:-"-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl
-Pkinesis-asl -Phive-thriftserver -Phive"}
+TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export
tools/fullClasspath" | grep jar | tail -n1)"
+OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES
"export oldDeps/fullClasspath" | grep jar | tail -n1)"
rm -f .generated-mima*
@@ -42,7 +42,7 @@ $JAVA_CMD \
-cp "$TOOLS_CLASSPATH:$OLD_DEPS_CLASSPATH" \
org.apache.spark.tools.GenerateMIMAIgnore
-echo -e "q\n" | build/sbt -Pscala-2.13 -mem 5632 -DcopyDependencies=false "$@"
mimaReportBinaryIssues | grep -v -e "info.*Resolving"
+echo -e "q\n" | build/sbt -mem 5632 -DcopyDependencies=false "$@"
mimaReportBinaryIssues | grep -v -e "info.*Resolving"
ret_val=$?
if [ $ret_val != 0 ]; then
diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb
index 9cb073ef1e0..ec9127e5165 100644
--- a/docs/_plugins/copy_api_dirs.rb
+++ b/docs/_plugins/copy_api_dirs.rb
@@ -26,8 +26,8 @@ if not (ENV['SKIP_API'] == '1')
curr_dir = pwd
cd("..")
- puts "Running 'build/sbt -Pscala-2.13 -Pkinesis-asl clean compile unidoc'
from " + pwd + "; this may take a few minutes..."
- system("build/sbt -Pscala-2.13 -Pkinesis-asl clean compile unidoc") ||
raise("Unidoc generation failed")
+ puts "Running 'build/sbt -Pkinesis-asl clean compile unidoc' from " + pwd
+ "; this may take a few minutes..."
+ system("build/sbt -Pkinesis-asl clean compile unidoc") || raise("Unidoc
generation failed")
puts "Moving back into docs dir."
cd("docs")
@@ -119,8 +119,8 @@ if not (ENV['SKIP_API'] == '1')
puts "Moving to project root and building API docs."
cd("..")
- puts "Running 'build/sbt -Pscala-2.13 clean package -Phive' from " + pwd +
"; this may take a few minutes..."
- system("build/sbt -Pscala-2.13 clean package -Phive") || raise("PySpark
doc generation failed")
+ puts "Running 'build/sbt clean package -Phive' from " + pwd + "; this may
take a few minutes..."
+ system("build/sbt clean package -Phive") || raise("PySpark doc generation
failed")
puts "Moving back into docs dir."
cd("docs")
@@ -165,8 +165,8 @@ if not (ENV['SKIP_API'] == '1')
puts "Moving to project root and building API docs."
cd("..")
- puts "Running 'build/sbt -Pscala-2.13 clean package -Phive' from " + pwd
+ "; this may take a few minutes..."
- system("build/sbt -Pscala-2.13 clean package -Phive") || raise("SQL doc
generation failed")
+ puts "Running 'build/sbt clean package -Phive' from " + pwd + "; this
may take a few minutes..."
+ system("build/sbt clean package -Phive") || raise("SQL doc generation
failed")
puts "Moving back into docs dir."
cd("docs")
diff --git a/docs/building-spark.md b/docs/building-spark.md
index b7d1bacea3c..ba994920440 100644
--- a/docs/building-spark.md
+++ b/docs/building-spark.md
@@ -261,6 +261,7 @@ or
./build/sbt docker-integration-tests/test
+<!---
## Change Scala Version
When other versions of Scala like 2.13 are supported, it will be possible to
build for that version.
@@ -275,6 +276,7 @@ Enable the profile (e.g. 2.13):
# For sbt
./build/sbt -Pscala-2.13 compile
+-->
## Running Jenkins tests with GitHub Enterprise
diff --git a/mllib/pom.xml b/mllib/pom.xml
index 70c116846f4..88400e7ba6a 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -91,12 +91,10 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- <!-- #endif scala-2.13 -->
<dependency>
<groupId>org.scalanlp</groupId>
<artifactId>breeze_${scala.binary.version}</artifactId>
diff --git a/pom.xml b/pom.xml
index 65aa656aca2..e1c88327a94 100644
--- a/pom.xml
+++ b/pom.xml
@@ -440,13 +440,11 @@
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
- <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
<version>1.0.4</version>
</dependency>
- <!-- #endif scala-2.13 -->
<dependency>
<groupId>com.twitter</groupId>
<artifactId>chill_${scala.binary.version}</artifactId>
@@ -3648,13 +3646,14 @@
</properties>
</profile>
+ <!--
+ SPARK-34774 Add this property to ensure change-scala-version.sh can
replace the public `scala.version`
+ property correctly.
+ -->
+ <!--
<profile>
<id>scala-2.13</id>
<properties>
- <!--
- SPARK-34774 Add this property to ensure change-scala-version.sh can
replace the public `scala.version`
- property correctly.
- -->
<scala.version>2.13.11</scala.version>
</properties>
<build>
@@ -3664,6 +3663,7 @@
</pluginManagement>
</build>
</profile>
+ -->
<!--
This is a profile to enable the use of the ASF snapshot and staging
repositories
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 48702f69a8a..8f2b9ccffeb 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -83,12 +83,10 @@
<artifactId>spark-sketch_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
- <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- <!-- #endif scala-2.13 -->
<dependency>
<groupId>org.scalacheck</groupId>
<artifactId>scalacheck_${scala.binary.version}</artifactId>
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index ab7d1c07ee2..b2b8398c9d5 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -89,12 +89,10 @@
<scope>test</scope>
</dependency>
- <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- <!-- #endif scala-2.13 -->
<dependency>
<groupId>org.apache.orc</groupId>
<artifactId>orc-core</artifactId>
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 4d7b6696b0c..920a9b67376 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -61,12 +61,10 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- <!-- #endif scala-2.13 -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index adcf0f55dcc..8b884233379 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -79,12 +79,10 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- <!-- #endif scala-2.13 -->
<dependency>
<groupId>${hive.group}</groupId>
<artifactId>hive-common</artifactId>
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 673f719f2cb..f409791f467 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -50,12 +50,10 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
- <!-- #if scala-2.13 -->
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
- <!-- #endif scala-2.13 -->
<!--
This spark-tags test-dep is needed even though it isn't used in this
module, otherwise testing-cmds that exclude
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
