This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 3ab2936 [SPARK-33667][SQL][2.4] Respect the `spark.sql.caseSensitive`
config while resolving partition spec in v1 `SHOW PARTITIONS`
3ab2936 is described below
commit 3ab293682c449aac73b43d24bcbf732379476f1f
Author: Max Gekk <[email protected]>
AuthorDate: Sun Dec 6 14:23:54 2020 -0800
[SPARK-33667][SQL][2.4] Respect the `spark.sql.caseSensitive` config while
resolving partition spec in v1 `SHOW PARTITIONS`
### What changes were proposed in this pull request?
Preprocess the partition spec passed to the V1 SHOW PARTITIONS
implementation `ShowPartitionsCommand`, and normalize the passed spec according
to the partition columns w.r.t the case sensitivity flag
**spark.sql.caseSensitive**.
### Why are the changes needed?
V1 SHOW PARTITIONS is case sensitive in fact, and doesn't respect the SQL
config **spark.sql.caseSensitive** which is false by default, for instance:
```sql
spark-sql> CREATE TABLE tbl1 (price int, qty int, year int, month int)
> USING parquet
> PARTITIONED BY (year, month);
spark-sql> INSERT INTO tbl1 PARTITION(year = 2015, month = 1) SELECT 1, 1;
spark-sql> SHOW PARTITIONS tbl1 PARTITION(YEAR = 2015, Month = 1);
Error in query: Non-partitioning column(s) [YEAR, Month] are specified for
SHOW PARTITIONS;
```
The `SHOW PARTITIONS` command must show the partition `year = 2015, month =
1` specified by `YEAR = 2015, Month = 1`.
### Does this PR introduce _any_ user-facing change?
Yes. After the changes, the command above works as expected:
```sql
spark-sql> SHOW PARTITIONS tbl1 PARTITION(YEAR = 2015, Month = 1);
year=2015/month=1
```
### How was this patch tested?
By running the affected test suites:
```
$ build/sbt -Phive-2.3 -Phive-thriftserver "test:testOnly
org.apache.spark.sql.hive.execution.HiveCatalogedDDLSuite"
```
Closes #30627 from MaxGekk/show-partitions-case-sensitivity-test-2.4.
Authored-by: Max Gekk <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark/sql/execution/command/tables.scala | 18 ++++++++---------
.../spark/sql/execution/command/DDLSuite.scala | 23 +++++++++++++++++++++-
.../sql/hive/execution/HiveCommandSuite.scala | 2 +-
3 files changed, 31 insertions(+), 12 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index 4a75bcb..b3e736a 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -956,20 +956,18 @@ case class ShowPartitionsCommand(
DDLUtils.verifyPartitionProviderIsHive(sparkSession, table, "SHOW
PARTITIONS")
/**
- * Validate the partitioning spec by making sure all the referenced
columns are
+ * Normalizes the partition spec w.r.t the partition columns and case
sensitivity settings,
+ * and validates the spec by making sure all the referenced columns are
* defined as partitioning columns in table definition. An
AnalysisException exception is
* thrown if the partitioning spec is invalid.
*/
- if (spec.isDefined) {
- val badColumns =
spec.get.keySet.filterNot(table.partitionColumnNames.contains)
- if (badColumns.nonEmpty) {
- val badCols = badColumns.mkString("[", ", ", "]")
- throw new AnalysisException(
- s"Non-partitioning column(s) $badCols are specified for SHOW
PARTITIONS")
- }
- }
+ val normalizedSpec = spec.map(partitionSpec =>
PartitioningUtils.normalizePartitionSpec(
+ partitionSpec,
+ table.partitionColumnNames,
+ table.identifier.quotedString,
+ sparkSession.sessionState.conf.resolver))
- val partNames = catalog.listPartitionNames(tableName, spec)
+ val partNames = catalog.listPartitionNames(tableName, normalizedSpec)
partNames.map(Row(_))
}
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index 9c55964..d48a950 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -2847,7 +2847,7 @@ abstract class DDLSuite extends QueryTest with
SQLTestUtils {
}
}
- test("SPARK-33588: case sensitivity of partition spec") {
+ test("SPARK-33588: case sensitivity of partition spec in SHOW TABLE") {
val t = "part_table"
withTable(t) {
sql(s"""
@@ -2867,6 +2867,27 @@ abstract class DDLSuite extends QueryTest with
SQLTestUtils {
}
}
}
+
+ test("SPARK-33667: case sensitivity of partition spec in SHOW PARTITIONS") {
+ val t = "part_table"
+ withTable(t) {
+ sql(s"""
+ |CREATE TABLE $t (price int, qty int, year int, month int)
+ |USING $dataSource
+ |PARTITIONED BY (year, month)""".stripMargin)
+ sql(s"INSERT INTO $t PARTITION(year = 2015, month = 1) SELECT 1, 1")
+ Seq(
+ true -> "PARTITION(year = 2015, month = 1)",
+ false -> "PARTITION(YEAR = 2015, Month = 1)"
+ ).foreach { case (caseSensitive, partitionSpec) =>
+ withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+ checkAnswer(
+ sql(s"SHOW PARTITIONS $t $partitionSpec"),
+ Row("year=2015/month=1"))
+ }
+ }
+ }
+ }
}
object FakeLocalFsFileSystem {
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
index 9147a98..3baa35e 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
@@ -419,7 +419,7 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils
with TestHiveSingleto
val message2 = intercept[AnalysisException] {
sql("SHOW PARTITIONS parquet_tab4 PARTITION(abcd=2015, xyz=1)")
}.getMessage
- assert(message2.contains("Non-partitioning column(s) [abcd, xyz] are
specified"))
+ assert(message2.contains("abcd is not a valid partition column"))
val message3 = intercept[AnalysisException] {
sql("SHOW PARTITIONS parquet_view1")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
