This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1f24b2d72ed6 [SPARK-44811][BUILD] Upgrade Guava to 33.2.1-jre
1f24b2d72ed6 is described below
commit 1f24b2d72ed6821a6cc6d1d22683d2f3ba2326a2
Author: Cheng Pan <[email protected]>
AuthorDate: Thu Sep 12 09:26:56 2024 -0700
[SPARK-44811][BUILD] Upgrade Guava to 33.2.1-jre
### What changes were proposed in this pull request?
This PR upgrades Spark's built-in Guava from 14 to 33.2.1-jre
Currently, Spark uses Guava 14 because the previous built-in Hive 2.3.9 is
incompatible with new Guava versions. HIVE-27560
(https://github.com/apache/hive/pull/4542) makes Hive 2.3.10 compatible with
Guava 14+ (thanks to LuciferYang)
### Why are the changes needed?
It's a long-standing issue, see prior discussions at
https://github.com/apache/spark/pull/35584,
https://github.com/apache/spark/pull/36231, and
https://github.com/apache/spark/pull/33989
### Does this PR introduce _any_ user-facing change?
Yes, some user-faced error messages changed.
### How was this patch tested?
GA passed.
Closes #42493 from pan3793/guava.
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
assembly/pom.xml | 2 +-
core/pom.xml | 1 +
dev/deps/spark-deps-hadoop-3-hive-2.3 | 7 ++++++-
pom.xml | 3 ++-
project/SparkBuild.scala | 2 +-
.../spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala | 2 +-
.../src/test/resources/sql-tests/results/ansi/interval.sql.out | 4 ++--
sql/core/src/test/resources/sql-tests/results/interval.sql.out | 4 ++--
8 files changed, 16 insertions(+), 9 deletions(-)
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 4b074a88dab4..01bd324efc11 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -123,7 +123,7 @@
<!--
Because we don't shade dependencies anymore, we need to restore Guava to
compile scope so
that the libraries Spark depend on have it available. We'll package the
version that Spark
- uses (14.0.1) which is not the same as Hadoop dependencies, but works.
+ uses which is not the same as Hadoop dependencies, but works.
-->
<dependency>
<groupId>com.google.guava</groupId>
diff --git a/core/pom.xml b/core/pom.xml
index 53d5ad71cebf..19f58940ed94 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -558,6 +558,7 @@
<include>org.eclipse.jetty:jetty-util</include>
<include>org.eclipse.jetty:jetty-server</include>
<include>com.google.guava:guava</include>
+ <include>com.google.guava:failureaccess</include>
<include>com.google.protobuf:*</include>
<!-- While shading new dependencies, please also update
LICENSE/NOTICE files in the
core/src/main/resources/META-INF if necessary -->
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3
b/dev/deps/spark-deps-hadoop-3-hive-2.3
index c89c92815d45..2db86ed229a0 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -33,6 +33,7 @@ breeze-macros_2.13/2.1.0//breeze-macros_2.13-2.1.0.jar
breeze_2.13/2.1.0//breeze_2.13-2.1.0.jar
bundle/2.24.6//bundle-2.24.6.jar
cats-kernel_2.13/2.8.0//cats-kernel_2.13-2.8.0.jar
+checker-qual/3.42.0//checker-qual-3.42.0.jar
chill-java/0.10.0//chill-java-0.10.0.jar
chill_2.13/0.10.0//chill_2.13-0.10.0.jar
commons-cli/1.9.0//commons-cli-1.9.0.jar
@@ -62,12 +63,14 @@ derby/10.16.1.1//derby-10.16.1.1.jar
derbyshared/10.16.1.1//derbyshared-10.16.1.1.jar
derbytools/10.16.1.1//derbytools-10.16.1.1.jar
dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
+error_prone_annotations/2.26.1//error_prone_annotations-2.26.1.jar
esdk-obs-java/3.20.4.2//esdk-obs-java-3.20.4.2.jar
+failureaccess/1.0.2//failureaccess-1.0.2.jar
flatbuffers-java/24.3.25//flatbuffers-java-24.3.25.jar
gcs-connector/hadoop3-2.2.21/shaded/gcs-connector-hadoop3-2.2.21-shaded.jar
gmetric4j/1.0.10//gmetric4j-1.0.10.jar
gson/2.11.0//gson-2.11.0.jar
-guava/14.0.1//guava-14.0.1.jar
+guava/33.2.1-jre//guava-33.2.1-jre.jar
hadoop-aliyun/3.4.0//hadoop-aliyun-3.4.0.jar
hadoop-annotations/3.4.0//hadoop-annotations-3.4.0.jar
hadoop-aws/3.4.0//hadoop-aws-3.4.0.jar
@@ -101,6 +104,7 @@ icu4j/75.1//icu4j-75.1.jar
ini4j/0.5.4//ini4j-0.5.4.jar
istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
ivy/2.5.2//ivy-2.5.2.jar
+j2objc-annotations/3.0.0//j2objc-annotations-3.0.0.jar
jackson-annotations/2.17.2//jackson-annotations-2.17.2.jar
jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/2.17.2//jackson-core-2.17.2.jar
@@ -184,6 +188,7 @@ lapack/3.0.3//lapack-3.0.3.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
libfb303/0.9.3//libfb303-0.9.3.jar
libthrift/0.16.0//libthrift-0.16.0.jar
+listenablefuture/9999.0-empty-to-avoid-conflict-with-guava//listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
log4j-1.2-api/2.22.1//log4j-1.2-api-2.22.1.jar
log4j-api/2.22.1//log4j-api-2.22.1.jar
log4j-core/2.22.1//log4j-core-2.22.1.jar
diff --git a/pom.xml b/pom.xml
index 6f5c9b63f86d..b1497c782685 100644
--- a/pom.xml
+++ b/pom.xml
@@ -195,7 +195,7 @@
<!-- org.apache.commons/commons-pool2/-->
<commons-pool2.version>2.12.0</commons-pool2.version>
<datanucleus-core.version>4.1.17</datanucleus-core.version>
- <guava.version>14.0.1</guava.version>
+ <guava.version>33.2.1-jre</guava.version>
<gson.version>2.11.0</gson.version>
<janino.version>3.1.9</janino.version>
<jersey.version>3.0.12</jersey.version>
@@ -3420,6 +3420,7 @@
<includes>
<include>org.spark-project.spark:unused</include>
<include>com.google.guava:guava</include>
+ <include>com.google.guava:failureaccess</include>
<include>org.jpmml:*</include>
</includes>
</artifactSet>
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 01d4ad50a22b..4a8214b2e20a 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -1051,7 +1051,7 @@ object KubernetesIntegrationTests {
* Overrides to work around sbt's dependency resolution being different from
Maven's.
*/
object DependencyOverrides {
- lazy val guavaVersion = sys.props.get("guava.version").getOrElse("14.0.1")
+ lazy val guavaVersion =
sys.props.get("guava.version").getOrElse("33.1.0-jre")
lazy val settings = Seq(
dependencyOverrides += "com.google.guava" % "guava" % guavaVersion,
dependencyOverrides += "xerces" % "xercesImpl" % "2.12.2",
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
index ff5ffe4e869a..7caf23490a0c 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
@@ -351,7 +351,7 @@ class IntervalExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
Seq(
(Period.ofMonths(2), Int.MaxValue) -> "overflow",
- (Period.ofMonths(Int.MinValue), 10d) -> "not in range",
+ (Period.ofMonths(Int.MinValue), 10d) -> "out of range",
(Period.ofMonths(-100), Float.NaN) -> "input is infinite or NaN",
(Period.ofMonths(200), Double.PositiveInfinity) -> "input is infinite or
NaN",
(Period.ofMonths(-200), Float.NegativeInfinity) -> "input is infinite or
NaN"
diff --git
a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
index 9e5c89045e51..b2f85835eb0d 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
@@ -2890,7 +2890,7 @@ SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1.0D
struct<>
-- !query output
java.lang.ArithmeticException
-not in range
+rounded value is out of range for input 2.147483648E9 and rounding mode HALF_UP
-- !query
@@ -2970,7 +2970,7 @@ SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO
SECOND) / -1.0D
struct<>
-- !query output
java.lang.ArithmeticException
-not in range
+rounded value is out of range for input 9.223372036854776E18 and rounding mode
HALF_UP
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index a4e167051767..5471dafaec8e 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -2713,7 +2713,7 @@ SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1.0D
struct<>
-- !query output
java.lang.ArithmeticException
-not in range
+rounded value is out of range for input 2.147483648E9 and rounding mode HALF_UP
-- !query
@@ -2793,7 +2793,7 @@ SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO
SECOND) / -1.0D
struct<>
-- !query output
java.lang.ArithmeticException
-not in range
+rounded value is out of range for input 9.223372036854776E18 and rounding mode
HALF_UP
-- !query
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
