This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/spark.git
commit a21a0769b20c5fc613eb1b75362b349ba39290a4 Author: beliefer <[email protected]> AuthorDate: Tue Mar 31 12:35:01 2020 +0900 [SPARK-31279][SQL][DOC] Add version information to the configuration of Hive ### What changes were proposed in this pull request? Add version information to the configuration of `Hive`. I sorted out some information show below. Item name | Since version | JIRA ID | Commit ID | Note -- | -- | -- | -- | -- spark.sql.hive.metastore.version | 1.4.0 | SPARK-6908 | 05454fd8aef75b129cbbd0288f5089c5259f4a15#diff-ff50aea397a607b79df9bec6f2a841db | spark.sql.hive.version | 1.1.1 | SPARK-3971 | 64945f868443fbc59cb34b34c16d782dda0fb63d#diff-12fa2178364a810b3262b30d8d48aa2d | spark.sql.hive.metastore.jars | 1.4.0 | SPARK-6908 | 05454fd8aef75b129cbbd0288f5089c5259f4a15#diff-ff50aea397a607b79df9bec6f2a841db | spark.sql.hive.convertMetastoreParquet | 1.1.1 | SPARK-2406 | cc4015d2fa3785b92e6ab079b3abcf17627f7c56#diff-ff50aea397a607b79df9bec6f2a841db | spark.sql.hive.convertMetastoreParquet.mergeSchema | 1.3.1 | SPARK-6575 | 778c87686af0c04df9dfe144b8f744f271a988ad#diff-ff50aea397a607b79df9bec6f2a841db | spark.sql.hive.convertMetastoreOrc | 2.0.0 | SPARK-14070 | 1e886159849e3918445d3fdc3c4cef86c6c1a236#diff-ff50aea397a607b79df9bec6f2a841db | spark.sql.hive.convertInsertingPartitionedTable | 3.0.0 | SPARK-28573 | d5688dc732890923c326f272b0c18c329a69459a#diff-842e3447fc453de26c706db1cac8f2c4 | spark.sql.hive.convertMetastoreCtas | 3.0.0 | SPARK-25271 | 5ad03607d1487e7ab3e3b6d00eef9c4028ed4975#diff-842e3447fc453de26c706db1cac8f2c4 | spark.sql.hive.metastore.sharedPrefixes | 1.4.0 | SPARK-7491 | a8556086d33cb993fab0ae2751e31455e6c664ab#diff-ff50aea397a607b79df9bec6f2a841db | spark.sql.hive.metastore.barrierPrefixes | 1.4.0 | SPARK-7491 | a8556086d33cb993fab0ae2751e31455e6c664ab#diff-ff50aea397a607b79df9bec6f2a841db | spark.sql.hive.thriftServer.async | 1.5.0 | SPARK-6964 | eb19d3f75cbd002f7e72ce02017a8de67f562792#diff-ff50aea397a607b79df9bec6f2a841db | ### Why are the changes needed? Supplemental configuration version information. ### Does this PR introduce any user-facing change? 'No'. ### How was this patch tested? Exists UT Closes #28042 from beliefer/add-version-to-hive-config. Authored-by: beliefer <[email protected]> Signed-off-by: HyukjinKwon <[email protected]> --- docs/sql-data-sources-hive-tables.md | 6 +++++- .../src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/sql-data-sources-hive-tables.md b/docs/sql-data-sources-hive-tables.md index 0054d46..22514cd 100644 --- a/docs/sql-data-sources-hive-tables.md +++ b/docs/sql-data-sources-hive-tables.md @@ -124,7 +124,7 @@ will compile against built-in Hive and use those classes for internal execution The following options can be used to configure the version of Hive that is used to retrieve metadata: <table class="table"> - <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr> + <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr> <tr> <td><code>spark.sql.hive.metastore.version</code></td> <td><code>2.3.6</code></td> @@ -132,6 +132,7 @@ The following options can be used to configure the version of Hive that is used Version of the Hive metastore. Available options are <code>0.12.0</code> through <code>2.3.6</code> and <code>3.0.0</code> through <code>3.1.2</code>. </td> + <td>1.4.0</td> </tr> <tr> <td><code>spark.sql.hive.metastore.jars</code></td> @@ -153,6 +154,7 @@ The following options can be used to configure the version of Hive that is used they are packaged with your application.</li> </ol> </td> + <td>1.4.0</td> </tr> <tr> <td><code>spark.sql.hive.metastore.sharedPrefixes</code></td> @@ -166,6 +168,7 @@ The following options can be used to configure the version of Hive that is used custom appenders that are used by log4j. </p> </td> + <td>1.4.0</td> </tr> <tr> <td><code>spark.sql.hive.metastore.barrierPrefixes</code></td> @@ -177,5 +180,6 @@ The following options can be used to configure the version of Hive that is used prefix that typically would be shared (i.e. <code>org.apache.spark.*</code>). </p> </td> + <td>1.4.0</td> </tr> </table> diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala index 9c4b8a5..3c20e68 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala @@ -65,6 +65,7 @@ private[spark] object HiveUtils extends Logging { .doc("Version of the Hive metastore. Available options are " + "<code>0.12.0</code> through <code>2.3.6</code> and " + "<code>3.0.0</code> through <code>3.1.2</code>.") + .version("1.4.0") .stringConf .createWithDefault(builtinHiveVersion) @@ -73,6 +74,7 @@ private[spark] object HiveUtils extends Logging { // already rely on this config. val FAKE_HIVE_VERSION = buildConf("spark.sql.hive.version") .doc(s"deprecated, please use ${HIVE_METASTORE_VERSION.key} to get the Hive version in Spark.") + .version("1.1.1") .stringConf .createWithDefault(builtinHiveVersion) @@ -89,12 +91,14 @@ private[spark] object HiveUtils extends Logging { | Use Hive jars of specified version downloaded from Maven repositories. | 3. A classpath in the standard format for both Hive and Hadoop. """.stripMargin) + .version("1.4.0") .stringConf .createWithDefault("builtin") val CONVERT_METASTORE_PARQUET = buildConf("spark.sql.hive.convertMetastoreParquet") .doc("When set to true, the built-in Parquet reader and writer are used to process " + "parquet tables created by using the HiveQL syntax, instead of Hive serde.") + .version("1.1.1") .booleanConf .createWithDefault(true) @@ -103,12 +107,14 @@ private[spark] object HiveUtils extends Logging { .doc("When true, also tries to merge possibly different but compatible Parquet schemas in " + "different Parquet data files. This configuration is only effective " + "when \"spark.sql.hive.convertMetastoreParquet\" is true.") + .version("1.3.1") .booleanConf .createWithDefault(false) val CONVERT_METASTORE_ORC = buildConf("spark.sql.hive.convertMetastoreOrc") .doc("When set to true, the built-in ORC reader and writer are used to process " + "ORC tables created by using the HiveQL syntax, instead of Hive serde.") + .version("2.0.0") .booleanConf .createWithDefault(true) @@ -118,6 +124,7 @@ private[spark] object HiveUtils extends Logging { "`spark.sql.hive.convertMetastoreOrc` is true, the built-in ORC/Parquet writer is used" + "to process inserting into partitioned ORC/Parquet tables created by using the HiveSQL " + "syntax.") + .version("3.0.0") .booleanConf .createWithDefault(true) @@ -126,6 +133,7 @@ private[spark] object HiveUtils extends Logging { "instead of Hive serde in CTAS. This flag is effective only if " + "`spark.sql.hive.convertMetastoreParquet` or `spark.sql.hive.convertMetastoreOrc` is " + "enabled respectively for Parquet and ORC formats") + .version("3.0.0") .booleanConf .createWithDefault(true) @@ -135,6 +143,7 @@ private[spark] object HiveUtils extends Logging { "that should be shared is JDBC drivers that are needed to talk to the metastore. Other " + "classes that need to be shared are those that interact with classes that are already " + "shared. For example, custom appenders that are used by log4j.") + .version("1.4.0") .stringConf .toSequence .createWithDefault(jdbcPrefixes) @@ -146,12 +155,14 @@ private[spark] object HiveUtils extends Logging { .doc("A comma separated list of class prefixes that should explicitly be reloaded for each " + "version of Hive that Spark SQL is communicating with. For example, Hive UDFs that are " + "declared in a prefix that typically would be shared (i.e. <code>org.apache.spark.*</code>).") + .version("1.4.0") .stringConf .toSequence .createWithDefault(Nil) val HIVE_THRIFT_SERVER_ASYNC = buildConf("spark.sql.hive.thriftServer.async") .doc("When set to true, Hive Thrift server executes SQL queries in an asynchronous way.") + .version("1.5.0") .booleanConf .createWithDefault(true) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
