This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 3f8a8b36d26 [SPARK-40798][SQL][FOLLOW-UP] Fix ansi mode in v2 ALTER TABLE PARTITION 3f8a8b36d26 is described below commit 3f8a8b36d264acdf688ea03033371b1171bfedf0 Author: ulysses-you <ulyssesyo...@gmail.com> AuthorDate: Thu Nov 3 19:28:51 2022 +0900 [SPARK-40798][SQL][FOLLOW-UP] Fix ansi mode in v2 ALTER TABLE PARTITION ### What changes were proposed in this pull request? Change to plain cast to `PartitioningUtils.castPartitionSpec` in convertToPartIdent. So the behavior can follow the `STORE_ASSIGNMENT_POLICY`. ### Why are the changes needed? Make v2 code path ALTER PARTITION follows `STORE_ASSIGNMENT_POLICY` in ansi mode. ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? remove the test `SPARK-40798: Alter partition should verify partition value - legacy`. This change has been convered at : `SPARK-40798: Alter partition should verify partition value` Closes #38449 from ulysses-you/SPARK-40798-follow. Authored-by: ulysses-you <ulyssesyo...@gmail.com> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- .../spark/sql/catalyst/analysis/ResolvePartitionSpec.scala | 9 +++++++-- .../sql/execution/command/v2/AlterTableAddPartitionSuite.scala | 4 +--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolvePartitionSpec.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolvePartitionSpec.scala index e09991cb2b9..90a502653d0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolvePartitionSpec.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolvePartitionSpec.scala @@ -25,8 +25,9 @@ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreePattern.COMMAND import org.apache.spark.sql.catalyst.util.CharVarcharUtils import org.apache.spark.sql.connector.catalog.SupportsPartitionManagement +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ -import org.apache.spark.sql.util.PartitioningUtils.{normalizePartitionSpec, requireExactMatchedPartitionSpec} +import org.apache.spark.sql.util.PartitioningUtils.{castPartitionSpec, normalizePartitionSpec, requireExactMatchedPartitionSpec} /** * Resolve [[UnresolvedPartitionSpec]] to [[ResolvedPartitionSpec]] in partition related commands. @@ -78,7 +79,11 @@ object ResolvePartitionSpec extends Rule[LogicalPlan] { val partValues = schema.map { part => val raw = partitionSpec.get(part.name).orNull val dt = CharVarcharUtils.replaceCharVarcharWithString(part.dataType) - Cast(Literal.create(raw, StringType), dt, Some(conf.sessionLocalTimeZone)).eval() + if (SQLConf.get.getConf(SQLConf.SKIP_TYPE_VALIDATION_ON_ALTER_PARTITION)) { + Cast(Literal.create(raw, StringType), dt, Some(conf.sessionLocalTimeZone)).eval() + } else { + castPartitionSpec(raw, dt, conf).eval() + } } InternalRow.fromSeq(partValues) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableAddPartitionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableAddPartitionSuite.scala index c33d9b0101a..a9ab11e483f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableAddPartitionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/AlterTableAddPartitionSuite.scala @@ -129,9 +129,7 @@ class AlterTableAddPartitionSuite withNamespaceAndTable("ns", "tbl") { t => sql(s"CREATE TABLE $t (c int) $defaultUsing PARTITIONED BY (p int)") - withSQLConf( - SQLConf.SKIP_TYPE_VALIDATION_ON_ALTER_PARTITION.key -> "true", - SQLConf.ANSI_ENABLED.key -> "false") { + withSQLConf(SQLConf.SKIP_TYPE_VALIDATION_ON_ALTER_PARTITION.key -> "true") { sql(s"ALTER TABLE $t ADD PARTITION (p='aaa')") checkPartitions(t, Map("p" -> defaultPartitionName)) sql(s"ALTER TABLE $t DROP PARTITION (p=null)") --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org