This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 55cd209 [SPARK-31522][SQL] Hive metastore client initialization
related configurations should be static
55cd209 is described below
commit 55cd20901e1e27af63985a0fee1ac900fdcfccb9
Author: Kent Yao <[email protected]>
AuthorDate: Thu Apr 23 15:07:44 2020 +0000
[SPARK-31522][SQL] Hive metastore client initialization related
configurations should be static
### What changes were proposed in this pull request?
HiveClient instance is cross-session, the following configurations which
are defined in HiveUtils and used to create it should be considered static:
1. spark.sql.hive.metastore.version - used to determine the hive version in
Spark
2. spark.sql.hive.metastore.jars - hive metastore related jars location
which is used by spark to create hive client
3. spark.sql.hive.metastore.sharedPrefixes and
spark.sql.hive.metastore.barrierPrefixes - package names of classes that are
shared or separated between SparkContextLoader and hive client class loader
Those are used only once when creating the hive metastore client. They
should be static in SQLConf for retrieving them correctly. We should avoid them
being changed by users with SET/RESET command.
Speaking of spark.sql.hive.version - the fake of the
spark.sql.hive.metastore.version, it is used by jdbc/thrift client for backward
compatibility.
### Why are the changes needed?
bugfix, these configurations should not be changed.
### Does this PR introduce any user-facing change?
Yes, the following set of configs are not allowed to change.
```
Seq("spark.sql.hive.metastore.version ",
"spark.sql.hive.metastore.jars",
"spark.sql.hive.metastore.sharedPrefixes",
"spark.sql.hive.metastore.barrierPrefixes")
```
### How was this patch tested?
add unit test
Closes #28302 from yaooqinn/SPARK-31522.
Authored-by: Kent Yao <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit 8dc2c0247be0be370d764653d5a68b7aa7948e39)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala | 11 +++++------
.../org/apache/spark/sql/hive/execution/SQLQuerySuite.scala | 10 ++++++++++
2 files changed, 15 insertions(+), 6 deletions(-)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
index d0f7988..04caf57 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
@@ -61,7 +61,7 @@ private[spark] object HiveUtils extends Logging {
/** The version of hive used internally by Spark SQL. */
val builtinHiveVersion: String = if (isHive23) hiveVersion else "1.2.1"
- val HIVE_METASTORE_VERSION = buildConf("spark.sql.hive.metastore.version")
+ val HIVE_METASTORE_VERSION =
buildStaticConf("spark.sql.hive.metastore.version")
.doc("Version of the Hive metastore. Available options are " +
"<code>0.12.0</code> through <code>2.3.7</code> and " +
"<code>3.0.0</code> through <code>3.1.2</code>.")
@@ -75,10 +75,9 @@ private[spark] object HiveUtils extends Logging {
val FAKE_HIVE_VERSION = buildConf("spark.sql.hive.version")
.doc(s"deprecated, please use ${HIVE_METASTORE_VERSION.key} to get the
Hive version in Spark.")
.version("1.1.1")
- .stringConf
- .createWithDefault(builtinHiveVersion)
+ .fallbackConf(HIVE_METASTORE_VERSION)
- val HIVE_METASTORE_JARS = buildConf("spark.sql.hive.metastore.jars")
+ val HIVE_METASTORE_JARS = buildStaticConf("spark.sql.hive.metastore.jars")
.doc(s"""
| Location of the jars that should be used to instantiate the
HiveMetastoreClient.
| This property can be one of three options: "
@@ -137,7 +136,7 @@ private[spark] object HiveUtils extends Logging {
.booleanConf
.createWithDefault(true)
- val HIVE_METASTORE_SHARED_PREFIXES =
buildConf("spark.sql.hive.metastore.sharedPrefixes")
+ val HIVE_METASTORE_SHARED_PREFIXES =
buildStaticConf("spark.sql.hive.metastore.sharedPrefixes")
.doc("A comma separated list of class prefixes that should be loaded using
the classloader " +
"that is shared between Spark SQL and a specific version of Hive. An
example of classes " +
"that should be shared is JDBC drivers that are needed to talk to the
metastore. Other " +
@@ -151,7 +150,7 @@ private[spark] object HiveUtils extends Logging {
private def jdbcPrefixes = Seq(
"com.mysql.jdbc", "org.postgresql", "com.microsoft.sqlserver",
"oracle.jdbc")
- val HIVE_METASTORE_BARRIER_PREFIXES =
buildConf("spark.sql.hive.metastore.barrierPrefixes")
+ val HIVE_METASTORE_BARRIER_PREFIXES =
buildStaticConf("spark.sql.hive.metastore.barrierPrefixes")
.doc("A comma separated list of class prefixes that should explicitly be
reloaded for each " +
"version of Hive that Spark SQL is communicating with. For example, Hive
UDFs that are " +
"declared in a prefix that typically would be shared (i.e.
<code>org.apache.spark.*</code>).")
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 138dcc5..e93f585 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -2493,6 +2493,16 @@ abstract class SQLQuerySuiteBase extends QueryTest with
SQLTestUtils with TestHi
}
}
}
+
+ test("SPARK-31522: hive metastore related configurations should be static") {
+ Seq("spark.sql.hive.metastore.version",
+ "spark.sql.hive.metastore.jars",
+ "spark.sql.hive.metastore.sharedPrefixes",
+ "spark.sql.hive.metastore.barrierPrefixes").foreach { key =>
+ val e = intercept[AnalysisException](sql(s"set $key=abc"))
+ assert(e.getMessage.contains("Cannot modify the value of a static
config"))
+ }
+ }
}
class SQLQuerySuite extends SQLQuerySuiteBase with
DisableAdaptiveExecutionSuite
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
