This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new d4b34d2898ec [SPARK-49843][SQL][3.5] Fix change comment on
char/varchar columns
d4b34d2898ec is described below
commit d4b34d2898ec983642a512db90f7afef4a1e8c88
Author: Stefan Kandic <[email protected]>
AuthorDate: Fri Oct 4 12:46:37 2024 -0700
[SPARK-49843][SQL][3.5] Fix change comment on char/varchar columns
### What changes were proposed in this pull request?
Fix the issue in `AlterTableChangeColumnCommand` where changing the comment
of a char/varchar column also tries to change the column type to string.
Backporting #48315 to 3.5
### Why are the changes needed?
Because the newColumn will always be a `StringType` even when the metadata
says that it was originally char/varchar.
### Does this PR introduce _any_ user-facing change?
Yes, the query will no longer fail when using this code path.
### How was this patch tested?
New query in golden files.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #48333 from stefankandic/branch-3.5.
Authored-by: Stefan Kandic <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../catalyst/analysis/ResolveSessionCatalog.scala | 10 +++++--
.../sql-tests/analyzer-results/charvarchar.sql.out | 12 ++++++++
.../resources/sql-tests/inputs/charvarchar.sql | 2 ++
.../sql-tests/results/charvarchar.sql.out | 32 ++++++++++++++++------
4 files changed, 45 insertions(+), 11 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index 0a86a043985e..2a92dc59f387 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -25,7 +25,7 @@ import
org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable
import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.util.{quoteIfNeeded, toPrettySQL,
ResolveDefaultColumns => DefaultCols}
+import org.apache.spark.sql.catalyst.util.{quoteIfNeeded, toPrettySQL,
CharVarcharUtils, ResolveDefaultColumns => DefaultCols}
import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns._
import org.apache.spark.sql.connector.catalog.{CatalogExtension,
CatalogManager, CatalogPlugin, CatalogV2Util, LookupCatalog,
SupportsNamespaces, V1Table}
import org.apache.spark.sql.connector.expressions.Transform
@@ -35,7 +35,7 @@ import
org.apache.spark.sql.execution.datasources.{CreateTable => CreateTableV1,
import org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2
import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
import org.apache.spark.sql.internal.connector.V1Function
-import org.apache.spark.sql.types.{MetadataBuilder, StructField, StructType}
+import org.apache.spark.sql.types.{MetadataBuilder, StringType, StructField,
StructType}
/**
* Converts resolved v2 commands to v1 if the catalog is the session catalog.
Since the v2 commands
@@ -84,7 +84,11 @@ class ResolveSessionCatalog(val catalogManager:
CatalogManager)
val colName = a.column.name(0)
val dataType = a.dataType.getOrElse {
table.schema.findNestedField(Seq(colName), resolver = conf.resolver)
- .map(_._2.dataType)
+ .map {
+ case (_, StructField(_, st: StringType, _, metadata)) =>
+ CharVarcharUtils.getRawType(metadata).getOrElse(st)
+ case (_, field) => field.dataType
+ }
.getOrElse {
throw
QueryCompilationErrors.alterColumnCannotFindColumnInV1TableError(
quoteIfNeeded(colName), table)
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/charvarchar.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/charvarchar.sql.out
index 6e72fd28686a..544d736b56b6 100644
--- a/sql/core/src/test/resources/sql-tests/analyzer-results/charvarchar.sql.out
+++ b/sql/core/src/test/resources/sql-tests/analyzer-results/charvarchar.sql.out
@@ -255,6 +255,18 @@ desc formatted char_part
DescribeTableCommand `spark_catalog`.`default`.`char_part`, true, [col_name#x,
data_type#x, comment#x]
+-- !query
+alter table char_part change column c1 comment 'char comment'
+-- !query analysis
+AlterTableChangeColumnCommand `spark_catalog`.`default`.`char_part`, c1,
StructField(c1,CharType(5),true)
+
+
+-- !query
+alter table char_part change column v1 comment 'varchar comment'
+-- !query analysis
+AlterTableChangeColumnCommand `spark_catalog`.`default`.`char_part`, v1,
StructField(v1,VarcharType(6),true)
+
+
-- !query
alter table char_part add partition (v2='ke', c2='nt') location 'loc1'
-- !query analysis
diff --git a/sql/core/src/test/resources/sql-tests/inputs/charvarchar.sql
b/sql/core/src/test/resources/sql-tests/inputs/charvarchar.sql
index 8117dec53f4a..be038e1083cd 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/charvarchar.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/charvarchar.sql
@@ -49,6 +49,8 @@ desc formatted char_tbl1;
create table char_part(c1 char(5), c2 char(2), v1 varchar(6), v2 varchar(2))
using parquet partitioned by (v2, c2);
desc formatted char_part;
+alter table char_part change column c1 comment 'char comment';
+alter table char_part change column v1 comment 'varchar comment';
alter table char_part add partition (v2='ke', c2='nt') location 'loc1';
desc formatted char_part;
diff --git a/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out
b/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out
index 888e8a942891..dd8bdc698ea7 100644
--- a/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out
@@ -543,6 +543,22 @@ Location [not included in
comparison]/{warehouse_dir}/char_part
Partition Provider Catalog
+-- !query
+alter table char_part change column c1 comment 'char comment'
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+alter table char_part change column v1 comment 'varchar comment'
+-- !query schema
+struct<>
+-- !query output
+
+
+
-- !query
alter table char_part add partition (v2='ke', c2='nt') location 'loc1'
-- !query schema
@@ -556,8 +572,8 @@ desc formatted char_part
-- !query schema
struct<col_name:string,data_type:string,comment:string>
-- !query output
-c1 char(5)
-v1 varchar(6)
+c1 char(5) char comment
+v1 varchar(6) varchar comment
v2 varchar(2)
c2 char(2)
# Partition Information
@@ -599,8 +615,8 @@ desc formatted char_part
-- !query schema
struct<col_name:string,data_type:string,comment:string>
-- !query output
-c1 char(5)
-v1 varchar(6)
+c1 char(5) char comment
+v1 varchar(6) varchar comment
v2 varchar(2)
c2 char(2)
# Partition Information
@@ -634,8 +650,8 @@ desc formatted char_part
-- !query schema
struct<col_name:string,data_type:string,comment:string>
-- !query output
-c1 char(5)
-v1 varchar(6)
+c1 char(5) char comment
+v1 varchar(6) varchar comment
v2 varchar(2)
c2 char(2)
# Partition Information
@@ -669,8 +685,8 @@ desc formatted char_part
-- !query schema
struct<col_name:string,data_type:string,comment:string>
-- !query output
-c1 char(5)
-v1 varchar(6)
+c1 char(5) char comment
+v1 varchar(6) varchar comment
v2 varchar(2)
c2 char(2)
# Partition Information
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
