This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 3818e537c749 [SPARK-53840][SQL][FOLLOWUP] Make SHOW TABLE EXTENDED AS
JSON match V2 metadata
3818e537c749 is described below
commit 3818e537c749a0b35781c16ad171ded8fb866d2a
Author: Linhong Liu <[email protected]>
AuthorDate: Tue Jul 21 21:00:37 2026 +0800
[SPARK-53840][SQL][FOLLOWUP] Make SHOW TABLE EXTENDED AS JSON match V2
metadata
### What changes were proposed in this pull request?
This follow-up updates `SHOW TABLE EXTENDED ... AS JSON` for V2 catalogs to
use catalog-provided summary metadata when producing extended JSON rows.
- For V2 relation catalogs, extended JSON now uses
`RelationCatalog#listRelationSummaries`, so relations such as views are
preserved.
- For V2 table catalogs, extended JSON now uses
`TableCatalog#listTableSummaries`.
- The extended JSON `type` field now comes from `TableSummary.tableType()`
instead of being hard-coded to `TABLE`.
- V2 command-suite coverage now checks the V2 summary table type and covers
a relation catalog returning both a table and a view.
### Why are the changes needed?
`SHOW TABLE EXTENDED` text output uses V2 summary metadata for table type
information, but the JSON path was falling back to identifier-only table
listing and hard-coding the type as `TABLE`. That loses V2 table type metadata,
and for relation catalogs it can also miss V2 views that are available through
relation summaries.
### Does this PR introduce _any_ user-facing change?
Yes. On unreleased Spark master, `SHOW TABLE EXTENDED ... AS JSON` for V2
catalogs now reports the catalog-provided table or view type metadata and
preserves V2 relation-catalog views in the JSON output.
### How was this patch tested?
Added regression coverage in `ShowTablesSuite`.
Also checked:
```
git diff --check apache/master..HEAD
```
GitHub PR checks passed on head `39f507910201d5dd08bbdb58125e1f7c0ac3809a`:
- `Build`
- `Notify test workflow`
Attempted local Spark SQL suite validation, but the environment could not
download required build dependencies:
```
./build/sbt "sql/testOnly
org.apache.spark.sql.execution.command.v2.ShowTablesSuite -- -z 'show table
extended as json returns relation catalog table and view types'"
./build/mvn -pl sql/core -am -DskipTests -DskipJavaTests -DskipScalaTests
-DskipMima -DskipCheckstyle test-compile
MAVEN_MIRROR_URL=https://repo.maven.apache.org/maven2 ./build/mvn -pl
sql/core -am -DskipTests -DskipJavaTests -DskipScalaTests -DskipMima
-DskipCheckstyle test-compile
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-5)
Closes #57362 from
linhongliu-db/task/spark-53840-show-table-extended-json-v2-metadata/implementation.
Authored-by: Linhong Liu <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit e6df45503058adc4e450f5476e5082d8dab5599b)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../execution/command/ShowTablesJsonCommand.scala | 22 +++++++++-----
.../sql/execution/command/v2/ShowTablesSuite.scala | 35 ++++++++++++++++++++++
2 files changed, 50 insertions(+), 7 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ShowTablesJsonCommand.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ShowTablesJsonCommand.scala
index 0dfc31705ae6..c52dc3de8fdd 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ShowTablesJsonCommand.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ShowTablesJsonCommand.scala
@@ -107,30 +107,38 @@ case class ShowTablesJsonCommand(
sparkSession: SparkSession,
catalog: TableCatalog,
ns: Seq[String]): JObject = {
- val identifiers = if (!isExtended) {
+ val relations = if (!isExtended) {
catalog match {
case mc: RelationCatalog =>
- mc.listRelationSummaries(ns.toArray).map(_.identifier())
- case _ => catalog.listTables(ns.toArray)
+ mc.listRelationSummaries(ns.toArray).map(s => (s.identifier(),
Option.empty[String]))
+ case _ =>
+ catalog.listTables(ns.toArray).map(ident => (ident,
Option.empty[String]))
}
} else {
- catalog.listTables(ns.toArray)
+ catalog match {
+ case mc: RelationCatalog =>
+ mc.listRelationSummaries(ns.toArray)
+ .map(s => (s.identifier(), Some(s.tableType())))
+ case _ =>
+ catalog.listTableSummaries(ns.toArray)
+ .map(s => (s.identifier(), Some(s.tableType())))
+ }
}
val pat = pattern.getOrElse("*")
- val filteredIdents = identifiers.filter { ident =>
+ val filteredRelations = relations.filter { case (ident, _) =>
StringUtils.filterPattern(Seq(ident.name()), pat).nonEmpty
}
val jsonRows = new ArrayBuffer[JObject]()
- filteredIdents.foreach { ident =>
+ filteredRelations.foreach { case (ident, tableType) =>
val nsArray = JArray(ident.namespace().map(JString(_)).toList)
val entry = if (isExtended) {
JObject(
"name" -> JString(ident.name()),
"catalog" -> JString(catalog.name()),
"namespace" -> nsArray,
- "type" -> JString("TABLE"),
+ "type" -> JString(tableType.get),
"isTemporary" -> JBool(false)
)
} else {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowTablesSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowTablesSuite.scala
index 775ca672f90b..e0888fee6ce2 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowTablesSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowTablesSuite.scala
@@ -17,7 +17,11 @@
package org.apache.spark.sql.execution.command.v2
+import org.json4s._
+import org.json4s.jackson.JsonMethods._
+
import org.apache.spark.sql.Row
+import org.apache.spark.sql.connector.catalog.{InMemoryRelationCatalog,
TableSummary}
import org.apache.spark.sql.execution.command
import org.apache.spark.util.Utils
@@ -26,6 +30,9 @@ import org.apache.spark.util.Utils
*/
class ShowTablesSuite extends command.ShowTablesSuiteBase with
CommandSuiteBase {
override def defaultNamespace: Seq[String] = Nil
+ // Extended JSON now uses TableSummary.tableType(), and
TableCatalog.listTableSummaries
+ // defaults a missing V2 table type property to
TableSummary.FOREIGN_TABLE_TYPE.
+ override def expectedTableTypeInJson: String =
TableSummary.FOREIGN_TABLE_TYPE
override def expectsSessionTempViews: Boolean = false
// The test fails for V1 catalog with the error:
@@ -73,6 +80,34 @@ class ShowTablesSuite extends command.ShowTablesSuiteBase
with CommandSuiteBase
}
}
+ test("show table extended as json returns relation catalog table and view
types") {
+ val relationCatalog = "show_table_extended_json_relation_catalog"
+ withSQLConf(s"spark.sql.catalog.$relationCatalog" ->
+ classOf[InMemoryRelationCatalog].getName) {
+ withNamespaceAndTable("ns", "tbl", relationCatalog) { t =>
+ sql(s"CREATE TABLE $t (id INT) $defaultUsing")
+ withView(s"$relationCatalog.ns.vw") {
+ sql(s"CREATE VIEW $relationCatalog.ns.vw AS SELECT 1 AS id")
+
+ val jsonStr = sql(
+ s"SHOW TABLE EXTENDED IN $relationCatalog.ns LIKE '*' AS JSON")
+ .collect()(0).getString(0)
+ val tables = (parse(jsonStr) \ "tables").asInstanceOf[JArray].arr
+ def entry(name: String): JValue = {
+ tables.find(e => (e \ "name").extract[String] == name)
+ .getOrElse(fail(s"Missing $name in SHOW TABLE EXTENDED AS JSON:
$tables"))
+ }
+
+ assert((entry("tbl") \ "type").extract[String] ===
+ TableSummary.FOREIGN_TABLE_TYPE)
+ assert((entry("vw") \ "type").extract[String] ===
+ TableSummary.VIEW_TABLE_TYPE)
+ assert(tables.map(e => (e \ "name").extract[String]).toSet ===
Set("tbl", "vw"))
+ }
+ }
+ }
+ }
+
override protected def extendedPartInNonPartedTableError(
catalog: String,
namespace: String,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
