This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new c0240ad [SPARK-37069][SQL] Properly fallback when
Hive.getWithoutRegisterFns is not available
c0240ad is described below
commit c0240ad792f6a987cd4f774e5b01059da95bad1b
Author: Chao Sun <sunc...@apple.com>
AuthorDate: Fri Oct 22 13:31:22 2021 +0900
[SPARK-37069][SQL] Properly fallback when Hive.getWithoutRegisterFns is not
available
Properly fallback to `Hive.get` when `Hive.getWithoutRegisterFns` is
unavailable to a Hive version.
In SPARK-35321 we switched to use the new method
`Hive.getWithoutRegisterFns` introduced by
[HIVE-21563](https://issues.apache.org/jira/browse/HIVE-21563). The code path
is supposed to only active for Hive versions that are >= 2.3.9. However, due to
how `HiveVersion` is initialized in `IsolatedClientLoader`, if users set
`spark.sql.hive.metastore.version` to `2.3.8`, Spark will still convert it to
"2.3.9" and thus will subsequently fail with `NoSuchMethodError`.
This fixes it by always fallback on `NoSuchMethodError`. By doing this we
are also able to support other Hive versions with `Hive.getWithoutRegisterFns`
implemented.
I manually tested via launching a Spark session with custom Hive version:
```
$SPARK_HOME/bin/spark-shell --conf spark.sql.hive.metastore.version=2.3.8
--conf spark.sql.hive.metastore.jars="/tmp/apache-hive-2.3.8-bin/lib/*
```
And then tried this command:
```
import org.apache.spark.sql.SparkSession
val spark = SparkSession.builder()
.master("local[*]")
.enableHiveSupport()
.config("spark.sql.hive.metastore.version", "2.3.8")
.config("spark.sql.hive.metastore.jars",
"/tmp/apache-hive-2.3.8-bin/lib/*")
.getOrCreate()
spark.sql("show tables").show
```
The command is failing before this PR, but working afterwards.
Closes #34360 from sunchao/SPARK-37069.
Authored-by: Chao Sun <sunc...@apple.com>
Signed-off-by: Hyukjin Kwon <gurwls...@apache.org>
(cherry picked from commit 39a0c224581ae5f221522cee1adeea5d2da73a08)
Signed-off-by: Hyukjin Kwon <gurwls...@apache.org>
---
.../apache/spark/sql/hive/client/HiveClientImpl.scala | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index edb1c12..d4362fa 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -57,11 +57,11 @@ import org.apache.spark.sql.catalyst.util.CharVarcharUtils
import org.apache.spark.sql.connector.catalog.SupportsNamespaces._
import org.apache.spark.sql.errors.{QueryCompilationErrors,
QueryExecutionErrors}
import org.apache.spark.sql.execution.QueryExecutionException
-import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils}
+import org.apache.spark.sql.hive.HiveExternalCatalog
import org.apache.spark.sql.hive.HiveExternalCatalog.DATASOURCE_SCHEMA
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
-import org.apache.spark.util.{CircularBuffer, Utils, VersionUtils}
+import org.apache.spark.util.{CircularBuffer, Utils}
/**
* A class that wraps the HiveClient and converts its responses to externally
visible classes.
@@ -201,12 +201,13 @@ private[hive] class HiveClientImpl(
}
private def getHive(conf: HiveConf): Hive = {
- VersionUtils.majorMinorPatchVersion(version.fullVersion).map {
- case (2, 3, v) if v >= 9 => Hive.getWithoutRegisterFns(conf)
- case _ => Hive.get(conf)
- }.getOrElse {
- throw QueryExecutionErrors.unsupportedHiveMetastoreVersionError(
- version.fullVersion, HiveUtils.HIVE_METASTORE_VERSION.key)
+ try {
+ Hive.getWithoutRegisterFns(conf)
+ } catch {
+ // SPARK-37069: not all Hive versions have the above method (e.g., Hive
2.3.9 has it but
+ // 2.3.8 don't), therefore here we fallback when encountering the
exception.
+ case _: NoSuchMethodError =>
+ Hive.get(conf)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
