This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 87d49cbc [SPARK-36381][SQL] Add case sensitive and case insensitive
compare for checking column name exist when alter table
87d49cbc is described below
commit 87d49cbcb1b9003763148dec6a3b067cf86f6ab3
Author: PengLei <[email protected]>
AuthorDate: Wed Aug 4 10:04:13 2021 +0900
[SPARK-36381][SQL] Add case sensitive and case insensitive compare for
checking column name exist when alter table
### What changes were proposed in this pull request?
Add the Resolver to `checkColumnNotExists` to check name exist in case
sensitive.
### Why are the changes needed?
At now the resolver is `_ == _` of `findNestedField` called by
`checkColumnNotExists`
Add `alter.conf.resolver` to it.
[SPARK-36381](https://issues.apache.org/jira/browse/SPARK-36381)
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Add ut tests
Closes #33618 from Peng-Lei/sensitive-cloumn-name.
Authored-by: PengLei <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../sql/catalyst/analysis/CheckAnalysis.scala | 9 +++++----
.../connector/V2CommandsCaseSensitivitySuite.scala | 22 ++++++++++++++++++++++
2 files changed, 27 insertions(+), 4 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index 09cd71b..043bf95 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -940,8 +940,9 @@ trait CheckAnalysis extends PredicateHelper with
LookupCatalog {
* Validates the options used for alter table commands after table and
columns are resolved.
*/
private def checkAlterTableCommand(alter: AlterTableCommand): Unit = {
- def checkColumnNotExists(op: String, fieldNames: Seq[String], struct:
StructType): Unit = {
- if (struct.findNestedField(fieldNames, includeCollections =
true).isDefined) {
+ def checkColumnNotExists(
+ op: String, fieldNames: Seq[String], struct: StructType, r: Resolver):
Unit = {
+ if (struct.findNestedField(fieldNames, includeCollections = true,
r).isDefined) {
alter.failAnalysis(s"Cannot $op column, because ${fieldNames.quoted} "
+
s"already exists in ${struct.treeString}")
}
@@ -950,7 +951,7 @@ trait CheckAnalysis extends PredicateHelper with
LookupCatalog {
alter match {
case AddColumns(table: ResolvedTable, colsToAdd) =>
colsToAdd.foreach { colToAdd =>
- checkColumnNotExists("add", colToAdd.name, table.schema)
+ checkColumnNotExists("add", colToAdd.name, table.schema,
alter.conf.resolver)
}
SchemaUtils.checkColumnNameDuplication(
colsToAdd.map(_.name.quoted),
@@ -958,7 +959,7 @@ trait CheckAnalysis extends PredicateHelper with
LookupCatalog {
alter.conf.resolver)
case RenameColumn(table: ResolvedTable, col: ResolvedFieldName, newName)
=>
- checkColumnNotExists("rename", col.path :+ newName, table.schema)
+ checkColumnNotExists("rename", col.path :+ newName, table.schema,
alter.conf.resolver)
case a @ AlterColumn(table: ResolvedTable, col: ResolvedFieldName, _, _,
_, _) =>
val fieldName = col.name.quoted
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala
index 1d6e0a8..0cc8d05 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala
@@ -249,6 +249,28 @@ class V2CommandsCaseSensitivitySuite extends
SharedSparkSession with AnalysisTes
expectErrorOnCaseSensitive = false)
}
+ test("SPARK-36381: Check column name exist case sensitive and insensitive
when add column") {
+ alterTableTest(
+ AddColumns(
+ table,
+ Seq(QualifiedColType(
+ None,
+ "ID",
+ LongType,
+ true,
+ None,
+ Some(UnresolvedFieldPosition(ColumnPosition.after("id")))))),
+ Seq("Cannot add column, because ID already exists in root"),
+ expectErrorOnCaseSensitive = false)
+ }
+
+ test("SPARK-36381: Check column name exist case sensitive and insensitive
when rename column") {
+ alterTableTest(
+ RenameColumn(table, UnresolvedFieldName(Array("id")), "DATA"),
+ Seq("Cannot rename column, because DATA already exists in root"),
+ expectErrorOnCaseSensitive = false)
+ }
+
test("AlterTable: drop column resolution") {
Seq(Array("ID"), Array("point", "X"), Array("POINT", "X"), Array("POINT",
"x")).foreach { ref =>
alterTableTest(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
