This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new f818a9f [SPARK-36213][SQL] Normalize PartitionSpec for Describe Table
Command with PartitionSpec
f818a9f is described below
commit f818a9f43bd6e36fffdf788f2dfc13cd01873d02
Author: Kent Yao <y...@apache.org>
AuthorDate: Thu Jul 22 00:52:31 2021 +0800
[SPARK-36213][SQL] Normalize PartitionSpec for Describe Table Command with
PartitionSpec
### What changes were proposed in this pull request?
This fixes a case sensitivity issue for desc table commands with partition
specified.
### Why are the changes needed?
bugfix
### Does this PR introduce _any_ user-facing change?
yes, but it's a bugfix
### How was this patch tested?
new tests
#### before
```
+-- !query
+DESC EXTENDED t PARTITION (C='Us', D=1)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Partition spec is invalid. The spec (C, D) must match the partition spec
(c, d) defined in table '`default`.`t`'
+
```
#### after
https://github.com/apache/spark/pull/33424/files#diff-554189c49950974a948f99fa9b7436f615052511660c6a0ae3062fa8ca0a327cR328
Closes #33424 from yaooqinn/SPARK-36213.
Authored-by: Kent Yao <y...@apache.org>
Signed-off-by: Kent Yao <y...@apache.org>
(cherry picked from commit 4cd6cfc773da726a90d41bfc590ea9188c17d5ae)
Signed-off-by: Kent Yao <y...@apache.org>
---
.../spark/sql/execution/command/tables.scala | 7 ++++-
.../test/resources/sql-tests/inputs/describe.sql | 2 ++
.../resources/sql-tests/results/describe.sql.out | 33 +++++++++++++++++++++-
3 files changed, 40 insertions(+), 2 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index f88200e..b557fd2 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -693,7 +693,12 @@ case class DescribeTableCommand(
s"DESC PARTITION is not allowed on a view: ${table.identifier}")
}
DDLUtils.verifyPartitionProviderIsHive(spark, metadata, "DESC PARTITION")
- val partition = catalog.getPartition(table, partitionSpec)
+ val normalizedPartSpec = PartitioningUtils.normalizePartitionSpec(
+ partitionSpec,
+ metadata.partitionSchema,
+ table.quotedString,
+ spark.sessionState.conf.resolver)
+ val partition = catalog.getPartition(table, normalizedPartSpec)
if (isExtended) describeFormattedDetailedPartitionInfo(table, metadata,
partition, result)
}
diff --git a/sql/core/src/test/resources/sql-tests/inputs/describe.sql
b/sql/core/src/test/resources/sql-tests/inputs/describe.sql
index a0ee932..deff5bb 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/describe.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/describe.sql
@@ -43,6 +43,8 @@ DESC EXTENDED t PARTITION (c='Us', d=1);
DESC FORMATTED t PARTITION (c='Us', d=1);
+DESC EXTENDED t PARTITION (C='Us', D=1);
+
-- NoSuchPartitionException: Partition not found in table
DESC t PARTITION (c='Us', d=2);
diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out
b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
index b0d650f..c14d312 100644
--- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 41
+-- Number of queries: 42
-- !query
@@ -325,6 +325,37 @@ Storage Properties [a=1, b=2]
-- !query
+DESC EXTENDED t PARTITION (C='Us', D=1)
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+a string
+b int
+c string
+d string
+# Partition Information
+# col_name data_type comment
+c string
+d string
+
+# Detailed Partition Information
+Database default
+Table t
+Partition Values [c=Us, d=1]
+Location [not included in comparison]/{warehouse_dir}/t/c=Us/d=1
+Storage Properties [a=1, b=2]
+Created Time [not included in comparison]
+Last Access [not included in comparison]
+
+# Storage Information
+Num Buckets 2
+Bucket Columns [`a`]
+Sort Columns [`b`]
+Location [not included in comparison]/{warehouse_dir}/t
+Storage Properties [a=1, b=2]
+
+
+-- !query
DESC t PARTITION (c='Us', d=2)
-- !query schema
struct<>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
