This is an automated email from the ASF dual-hosted git repository.
pan3793 pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new eb8956e5ac6a [SPARK-58260][SQL] Add config to render Hive DDL in SHOW
CREATE TABLE for Hive tables
eb8956e5ac6a is described below
commit eb8956e5ac6aa2ee9d9c30632062352a4dd3550d
Author: Cheng Pan <[email protected]>
AuthorDate: Thu Jul 23 23:24:57 2026 +0800
[SPARK-58260][SQL] Add config to render Hive DDL in SHOW CREATE TABLE for
Hive tables
### What changes were proposed in this pull request?
Add a new config `spark.sql.hive.showCreateTableAsSerde` (default `false`).
When enabled,
plain `SHOW CREATE TABLE` on a Hive serde table routes to the existing
`ShowCreateTableAsSerdeCommand` and renders Hive DDL, instead of converting
to Spark DDL.
### Why are the changes needed?
We get some reports from customers: they are confused that `CREATE TABLE
... STORED AS Parquet` but `SHOW CREATE TABLE ...` displays `CREATE TABLE ...
USING Parquet`.
Background: SPARK-27946 (#24938) deliberately changed plain `SHOW CREATE
TABLE` to always emit Spark DDL, to push and simplify migration from Hive to
Spark. `AS SERDE` was introduced in the same PR as the intended escape hatch
for users who still need Hive DDL, and the failure modes were intentional:
transactional Hive tables error out by design, and tables whose serde has no
Spark data source mapping (e.g. RCFILE) are meant to be served by `SHOW CREATE
TABLE AS SERDE`.
This PR keeps that direction intact: the default behavior is unchanged, and
`AS SERDE` remains the per-query escape hatch. It only adds a session-level
toggle that turns "append `AS SERDE` every time" into a config, which helps
users and integrations (e.g. BI tools, metastore sync jobs) that cannot easily
rewrite the SQL they issue.
### Does this PR introduce _any_ user-facing change?
Yes. New config `spark.sql.hive.showCreateTableAsSerde`, default `false`
(no behavior
change). When `true`, `SHOW CREATE TABLE` on Hive serde tables returns Hive
DDL; data
source tables and views are unaffected.
### How was this patch tested?
Added tests to `o.a.s.sql.hive.execution.command.ShowCreateTableSuite`
covering: Hive
table (matches `AS SERDE` output), data source table and view (unaffected),
RCFILE and
transactional Hive tables (Hive DDL rendered instead of erroring).
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: KIMI K3
Closes #57429 from pan3793/SPARK-58260.
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
(cherry picked from commit 0790f4d6a4cc2fce2f3a84e78b43f41856796289)
Signed-off-by: Cheng Pan <[email protected]>
---
.../org/apache/spark/sql/internal/SQLConf.scala | 13 +++
.../catalyst/analysis/ResolveSessionCatalog.scala | 7 +-
.../execution/command/ShowCreateTableSuite.scala | 93 +++++++++++++++++++++-
3 files changed, 111 insertions(+), 2 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index ea4c2ffc1559..965b80620968 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2079,6 +2079,16 @@ object SQLConf {
.intConf
.createOptional
+ val HIVE_TABLE_SHOW_CREATE_TABLE_AS_SERDE =
+ buildConf("spark.sql.hive.showCreateTableAsSerde")
+ .doc("Whether to render the SHOW CREATE TABLE (without `AS SERDE`)
statement of Hive table " +
+ "in the Hive DDL format. This is only for display purpose and does not
affect how Spark " +
+ "reads or writes Hive tables.")
+ .version("4.3.0")
+ .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE)
+ .booleanConf
+ .createWithDefault(false)
+
val OPTIMIZER_METADATA_ONLY = buildConf("spark.sql.optimizer.metadataOnly")
.internal()
.doc("When true, enable the metadata-only query optimization that use the
table's metadata " +
@@ -8483,6 +8493,9 @@ class SQLConf extends Serializable with Logging with
SqlApiConf {
def gatherFastStats: Boolean = getConf(GATHER_FASTSTAT)
+ def hiveTableShowCreateTableAsSerde: Boolean =
+ getConf(HIVE_TABLE_SHOW_CREATE_TABLE_AS_SERDE)
+
def optimizerMetadataOnly: Boolean = getConf(OPTIMIZER_METADATA_ONLY)
def wholeStageEnabled: Boolean = getConf(WHOLESTAGE_CODEGEN_ENABLED)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index 2ed873427e28..01f39e60afd3 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -455,7 +455,12 @@ class ResolveSessionCatalog(val catalogManager:
CatalogManager)
isOverwrite,
partition)
- case ShowCreateTable(ResolvedV1TableOrViewIdentifier(ident), asSerde,
output) if asSerde =>
+ case ShowCreateTable(ResolvedTable(catalog, _, t: V1Table, _), asSerde,
output)
+ if supportsV1Command(catalog) && (asSerde ||
+ (conf.hiveTableShowCreateTableAsSerde &&
DDLUtils.isHiveTable(t.catalogTable))) =>
+ ShowCreateTableAsSerdeCommand(t.catalogTable.identifier, output)
+
+ case ShowCreateTable(ResolvedViewIdentifier(ident), asSerde, output) if
asSerde =>
ShowCreateTableAsSerdeCommand(ident, output)
// If target is view, force use v1 command
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowCreateTableSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowCreateTableSuite.scala
index 8e654d28cd03..7502b1815aec 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowCreateTableSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowCreateTableSuite.scala
@@ -22,7 +22,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogUtils}
import org.apache.spark.sql.catalyst.util.escapeSingleQuotedString
import org.apache.spark.sql.execution.command.v1
-import org.apache.spark.sql.internal.HiveSerDe
+import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
/**
* The class contains tests for the `SHOW CREATE TABLE` command to check V1
Hive external
@@ -444,4 +444,95 @@ class ShowCreateTableSuite extends
v1.ShowCreateTableSuiteBase with CommandSuite
)
}
}
+
+ test("plain SHOW CREATE TABLE returns Hive DDL for hive table when the
config is on") {
+ withNamespaceAndTable(ns, table) { t =>
+ sql(
+ s"""CREATE TABLE $t (
+ | c1 INT COMMENT 'bla',
+ | c2 STRING
+ |) STORED AS PARQUET
+ """.stripMargin
+ )
+ val hiveDDL = getShowCreateDDL(t, true)
+ // By default, plain SHOW CREATE TABLE converts the Hive table to Spark
DDL.
+ assert(getShowCreateDDL(t) !== hiveDDL)
+
+ withSQLConf(SQLConf.HIVE_TABLE_SHOW_CREATE_TABLE_AS_SERDE.key -> "true")
{
+ assert(getShowCreateDDL(t) === hiveDDL)
+ assert(hiveDDL.mkString(" ").contains(
+ "ROW FORMAT SERDE
'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'"))
+ }
+ }
+ }
+
+ test("plain SHOW CREATE TABLE still returns Spark DDL for data source table
with config on") {
+ withSQLConf(SQLConf.HIVE_TABLE_SHOW_CREATE_TABLE_AS_SERDE.key -> "true") {
+ withNamespaceAndTable(ns, table) { t =>
+ sql(
+ s"""CREATE TABLE $t (
+ | c1 STRING COMMENT 'bla',
+ | c2 STRING
+ |) USING orc
+ """.stripMargin
+ )
+ // Should not fail with
UNSUPPORTED_SHOW_CREATE_TABLE.ON_DATA_SOURCE_TABLE_WITH_AS_SERDE.
+ assert(getShowCreateDDL(t).contains("USING orc"))
+ }
+ }
+ }
+
+ test("plain SHOW CREATE TABLE still returns Spark DDL for view when the
config is on") {
+ withSQLConf(SQLConf.HIVE_TABLE_SHOW_CREATE_TABLE_AS_SERDE.key -> "true") {
+ withView("v1") {
+ sql("CREATE VIEW v1 AS SELECT 1 AS c1")
+ assert(sql("SHOW CREATE TABLE
v1").head().getString(0).startsWith("CREATE VIEW"))
+ }
+ }
+ }
+
+ test("plain SHOW CREATE TABLE returns Hive DDL for unsupported serde when
the config is on") {
+ withSQLConf(SQLConf.HIVE_TABLE_SHOW_CREATE_TABLE_AS_SERDE.key -> "true") {
+ withTable("t1") {
+ sql(
+ s"""
+ |CREATE TABLE t1 (
+ | c1 INT COMMENT 'bla',
+ | c2 STRING
+ |) STORED AS RCFILE
+ """.stripMargin
+ )
+ // Without the config, this fails with
+ // UNSUPPORTED_SHOW_CREATE_TABLE.WITH_UNSUPPORTED_SERDE_CONFIGURATION
because RCFILE
+ // cannot be mapped to a Spark data source.
+ val hiveDDL = getShowCreateDDL("t1", true)
+ assert(getShowCreateDDL("t1") === hiveDDL)
+ assert(hiveDDL.mkString(" ").contains(
+ "INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat'"))
+ }
+ }
+ }
+
+ test("plain SHOW CREATE TABLE returns Hive DDL for transactional hive table
with config on") {
+ withSQLConf(SQLConf.HIVE_TABLE_SHOW_CREATE_TABLE_AS_SERDE.key -> "true") {
+ withTable("t1") {
+ sql(
+ s"""
+ |CREATE TABLE t1 (
+ | c1 STRING COMMENT 'bla',
+ | c2 STRING
+ |)
+ |TBLPROPERTIES ('transactional' = 'true')
+ |CLUSTERED BY (c1) INTO 10 BUCKETS
+ |STORED AS ORC
+ """.stripMargin
+ )
+ // Without the config, this fails with
+ // UNSUPPORTED_SHOW_CREATE_TABLE.ON_TRANSACTIONAL_HIVE_TABLE.
+ val showDDL = getShowCreateDDL("t1").mkString(" ")
+ assert(showDDL.contains("ROW FORMAT SERDE"))
+ assert(showDDL.contains("'transactional' = 'true'"))
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
