This is an automated email from the ASF dual-hosted git repository. wenchen pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push: new ebe4252e415 [SPARK-39149][SQL] SHOW DATABASES command should not quote database names under legacy mode ebe4252e415 is described below commit ebe4252e415e6afdf888e21d0b89ab744fd2dac7 Author: Wenchen Fan <wenc...@databricks.com> AuthorDate: Thu May 12 11:18:18 2022 +0800 [SPARK-39149][SQL] SHOW DATABASES command should not quote database names under legacy mode ### What changes were proposed in this pull request? This is a bug of the command legacy mode as it does not fully restore to the legacy behavior. The legacy v1 SHOW DATABASES command does not quote the database names. This PR fixes it. ### Why are the changes needed? bug fix ### Does this PR introduce _any_ user-facing change? no change by default, unless people turn on legacy mode, in which case SHOW DATABASES common won't quote the database names. ### How was this patch tested? new tests Closes #36508 from cloud-fan/regression. Authored-by: Wenchen Fan <wenc...@databricks.com> Signed-off-by: Wenchen Fan <wenc...@databricks.com> (cherry picked from commit 3094e495095635f6c9e83f4646d3321c2a9311f4) Signed-off-by: Wenchen Fan <wenc...@databricks.com> --- .../execution/datasources/v2/ShowNamespacesExec.scala | 11 ++++++++++- .../sql/execution/command/ShowNamespacesSuiteBase.scala | 17 +++++++++++++++++ .../sql/execution/command/v1/ShowNamespacesSuite.scala | 13 +++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala index 9dafbd79a52..c55c7b9f985 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala @@ -42,8 +42,17 @@ case class ShowNamespacesExec( catalog.listNamespaces() } + // Please refer to the rule `KeepLegacyOutputs` for details about legacy command. + // The legacy SHOW DATABASES command does not quote the database names. + val isLegacy = output.head.name == "databaseName" + val namespaceNames = if (isLegacy && namespaces.forall(_.length == 1)) { + namespaces.map(_.head) + } else { + namespaces.map(_.quoted) + } + val rows = new ArrayBuffer[InternalRow]() - namespaces.map(_.quoted).map { ns => + namespaceNames.map { ns => if (pattern.map(StringUtils.filterPattern(Seq(ns), _).nonEmpty).getOrElse(true)) { rows += toCatalystRow(ns) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowNamespacesSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowNamespacesSuiteBase.scala index b3693845c3b..80e545f6e3c 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowNamespacesSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowNamespacesSuiteBase.scala @@ -42,6 +42,9 @@ trait ShowNamespacesSuiteBase extends QueryTest with DDLCommandTestUtils { protected def builtinTopNamespaces: Seq[String] = Seq.empty protected def isCasePreserving: Boolean = true + protected def createNamespaceWithSpecialName(ns: String): Unit = { + sql(s"CREATE NAMESPACE $catalog.`$ns`") + } test("default namespace") { withSQLConf(SQLConf.DEFAULT_CATALOG.key -> catalog) { @@ -124,6 +127,20 @@ trait ShowNamespacesSuiteBase extends QueryTest with DDLCommandTestUtils { } } + test("SPARK-39149: keep the legacy no-quote behavior") { + Seq(true, false).foreach { legacy => + withSQLConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA.key -> legacy.toString) { + withNamespace(s"$catalog.`123`") { + createNamespaceWithSpecialName("123") + val res = if (legacy) "123" else "`123`" + checkAnswer( + sql(s"SHOW NAMESPACES IN $catalog"), + (res +: builtinTopNamespaces).map(Row(_))) + } + } + } + } + test("case sensitivity of the pattern string") { Seq(true, false).foreach { caseSensitive => withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowNamespacesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowNamespacesSuite.scala index a1b32e42ae2..b65a9acb656 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowNamespacesSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowNamespacesSuite.scala @@ -18,7 +18,9 @@ package org.apache.spark.sql.execution.command.v1 import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.catalyst.catalog.CatalogDatabase import org.apache.spark.sql.execution.command +import org.apache.spark.util.Utils /** * This base suite contains unified tests for the `SHOW NAMESPACES` and `SHOW DATABASES` commands @@ -31,6 +33,17 @@ import org.apache.spark.sql.execution.command trait ShowNamespacesSuiteBase extends command.ShowNamespacesSuiteBase { override protected def builtinTopNamespaces: Seq[String] = Seq("default") + override protected def createNamespaceWithSpecialName(ns: String): Unit = { + // Call `ExternalCatalog` directly to bypass the database name validation in `SessionCatalog`. + spark.sharedState.externalCatalog.createDatabase( + CatalogDatabase( + name = ns, + description = "", + locationUri = Utils.createTempDir().toURI, + properties = Map.empty), + ignoreIfExists = false) + } + test("IN namespace doesn't exist") { val errMsg = intercept[AnalysisException] { sql("SHOW NAMESPACES in dummy") --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org