This is an automated email from the ASF dual-hosted git repository. wenchen pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push: new 55cd209 [SPARK-31522][SQL] Hive metastore client initialization related configurations should be static 55cd209 is described below commit 55cd20901e1e27af63985a0fee1ac900fdcfccb9 Author: Kent Yao <yaooq...@hotmail.com> AuthorDate: Thu Apr 23 15:07:44 2020 +0000 [SPARK-31522][SQL] Hive metastore client initialization related configurations should be static ### What changes were proposed in this pull request? HiveClient instance is cross-session, the following configurations which are defined in HiveUtils and used to create it should be considered static: 1. spark.sql.hive.metastore.version - used to determine the hive version in Spark 2. spark.sql.hive.metastore.jars - hive metastore related jars location which is used by spark to create hive client 3. spark.sql.hive.metastore.sharedPrefixes and spark.sql.hive.metastore.barrierPrefixes - package names of classes that are shared or separated between SparkContextLoader and hive client class loader Those are used only once when creating the hive metastore client. They should be static in SQLConf for retrieving them correctly. We should avoid them being changed by users with SET/RESET command. Speaking of spark.sql.hive.version - the fake of the spark.sql.hive.metastore.version, it is used by jdbc/thrift client for backward compatibility. ### Why are the changes needed? bugfix, these configurations should not be changed. ### Does this PR introduce any user-facing change? Yes, the following set of configs are not allowed to change. ``` Seq("spark.sql.hive.metastore.version ", "spark.sql.hive.metastore.jars", "spark.sql.hive.metastore.sharedPrefixes", "spark.sql.hive.metastore.barrierPrefixes") ``` ### How was this patch tested? add unit test Closes #28302 from yaooqinn/SPARK-31522. Authored-by: Kent Yao <yaooq...@hotmail.com> Signed-off-by: Wenchen Fan <wenc...@databricks.com> (cherry picked from commit 8dc2c0247be0be370d764653d5a68b7aa7948e39) Signed-off-by: Wenchen Fan <wenc...@databricks.com> --- .../src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala | 11 +++++------ .../org/apache/spark/sql/hive/execution/SQLQuerySuite.scala | 10 ++++++++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala index d0f7988..04caf57 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala @@ -61,7 +61,7 @@ private[spark] object HiveUtils extends Logging { /** The version of hive used internally by Spark SQL. */ val builtinHiveVersion: String = if (isHive23) hiveVersion else "1.2.1" - val HIVE_METASTORE_VERSION = buildConf("spark.sql.hive.metastore.version") + val HIVE_METASTORE_VERSION = buildStaticConf("spark.sql.hive.metastore.version") .doc("Version of the Hive metastore. Available options are " + "<code>0.12.0</code> through <code>2.3.7</code> and " + "<code>3.0.0</code> through <code>3.1.2</code>.") @@ -75,10 +75,9 @@ private[spark] object HiveUtils extends Logging { val FAKE_HIVE_VERSION = buildConf("spark.sql.hive.version") .doc(s"deprecated, please use ${HIVE_METASTORE_VERSION.key} to get the Hive version in Spark.") .version("1.1.1") - .stringConf - .createWithDefault(builtinHiveVersion) + .fallbackConf(HIVE_METASTORE_VERSION) - val HIVE_METASTORE_JARS = buildConf("spark.sql.hive.metastore.jars") + val HIVE_METASTORE_JARS = buildStaticConf("spark.sql.hive.metastore.jars") .doc(s""" | Location of the jars that should be used to instantiate the HiveMetastoreClient. | This property can be one of three options: " @@ -137,7 +136,7 @@ private[spark] object HiveUtils extends Logging { .booleanConf .createWithDefault(true) - val HIVE_METASTORE_SHARED_PREFIXES = buildConf("spark.sql.hive.metastore.sharedPrefixes") + val HIVE_METASTORE_SHARED_PREFIXES = buildStaticConf("spark.sql.hive.metastore.sharedPrefixes") .doc("A comma separated list of class prefixes that should be loaded using the classloader " + "that is shared between Spark SQL and a specific version of Hive. An example of classes " + "that should be shared is JDBC drivers that are needed to talk to the metastore. Other " + @@ -151,7 +150,7 @@ private[spark] object HiveUtils extends Logging { private def jdbcPrefixes = Seq( "com.mysql.jdbc", "org.postgresql", "com.microsoft.sqlserver", "oracle.jdbc") - val HIVE_METASTORE_BARRIER_PREFIXES = buildConf("spark.sql.hive.metastore.barrierPrefixes") + val HIVE_METASTORE_BARRIER_PREFIXES = buildStaticConf("spark.sql.hive.metastore.barrierPrefixes") .doc("A comma separated list of class prefixes that should explicitly be reloaded for each " + "version of Hive that Spark SQL is communicating with. For example, Hive UDFs that are " + "declared in a prefix that typically would be shared (i.e. <code>org.apache.spark.*</code>).") diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 138dcc5..e93f585 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -2493,6 +2493,16 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi } } } + + test("SPARK-31522: hive metastore related configurations should be static") { + Seq("spark.sql.hive.metastore.version", + "spark.sql.hive.metastore.jars", + "spark.sql.hive.metastore.sharedPrefixes", + "spark.sql.hive.metastore.barrierPrefixes").foreach { key => + val e = intercept[AnalysisException](sql(s"set $key=abc")) + assert(e.getMessage.contains("Cannot modify the value of a static config")) + } + } } class SQLQuerySuite extends SQLQuerySuiteBase with DisableAdaptiveExecutionSuite --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org