This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 9f1d92050f2e [SPARK-49843][SQL][3.4] Fix change comment on
char/varchar columns
9f1d92050f2e is described below
commit 9f1d92050f2e6ca22d03a278565371f6cb6d7b7b
Author: Stefan Kandic <[email protected]>
AuthorDate: Fri Oct 4 12:48:54 2024 -0700
[SPARK-49843][SQL][3.4] Fix change comment on char/varchar columns
### What changes were proposed in this pull request?
Fix the issue in `AlterTableChangeColumnCommand` where changing the comment
of a char/varchar column also tries to change the column type to string.
Backporting #48315 to 3.4
### Why are the changes needed?
Because the newColumn will always be a `StringType` even when the metadata
says that it was originally char/varchar.
### Does this PR introduce _any_ user-facing change?
Yes, the query will no longer fail when using this code path.
### How was this patch tested?
New query in golden files.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #48334 from stefankandic/vcharFix.
Authored-by: Stefan Kandic <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../catalyst/analysis/ResolveSessionCatalog.scala | 10 +++++--
.../resources/sql-tests/inputs/charvarchar.sql | 2 ++
.../sql-tests/results/charvarchar.sql.out | 32 ++++++++++++++++------
3 files changed, 33 insertions(+), 11 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index e5a36394a44c..2f93a7b00d16 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -25,7 +25,7 @@ import
org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable
import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.util.{quoteIfNeeded, toPrettySQL,
ResolveDefaultColumns => DefaultCols}
+import org.apache.spark.sql.catalyst.util.{quoteIfNeeded, toPrettySQL,
CharVarcharUtils, ResolveDefaultColumns => DefaultCols}
import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns._
import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogV2Util,
LookupCatalog, SupportsNamespaces, V1Table}
import org.apache.spark.sql.connector.expressions.Transform
@@ -35,7 +35,7 @@ import
org.apache.spark.sql.execution.datasources.{CreateTable => CreateTableV1,
import org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2
import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
import org.apache.spark.sql.internal.connector.V1Function
-import org.apache.spark.sql.types.{MetadataBuilder, StructField, StructType}
+import org.apache.spark.sql.types.{MetadataBuilder, StringType, StructField,
StructType}
/**
* Converts resolved v2 commands to v1 if the catalog is the session catalog.
Since the v2 commands
@@ -84,7 +84,11 @@ class ResolveSessionCatalog(val catalogManager:
CatalogManager)
val colName = a.column.name(0)
val dataType = a.dataType.getOrElse {
table.schema.findNestedField(Seq(colName), resolver = conf.resolver)
- .map(_._2.dataType)
+ .map {
+ case (_, StructField(_, st: StringType, _, metadata)) =>
+ CharVarcharUtils.getRawType(metadata).getOrElse(st)
+ case (_, field) => field.dataType
+ }
.getOrElse {
throw
QueryCompilationErrors.alterColumnCannotFindColumnInV1TableError(
quoteIfNeeded(colName), table)
diff --git a/sql/core/src/test/resources/sql-tests/inputs/charvarchar.sql
b/sql/core/src/test/resources/sql-tests/inputs/charvarchar.sql
index b62cbf643232..37019a6d4ef9 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/charvarchar.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/charvarchar.sql
@@ -49,6 +49,8 @@ desc formatted char_tbl1;
create table char_part(c1 char(5), c2 char(2), v1 varchar(6), v2 varchar(2))
using parquet partitioned by (v2, c2);
desc formatted char_part;
+alter table char_part change column c1 comment 'char comment';
+alter table char_part change column v1 comment 'varchar comment';
alter table char_part add partition (v2='ke', c2='nt') location 'loc1';
desc formatted char_part;
diff --git a/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out
b/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out
index ae88227121b6..499ca1b9ab89 100644
--- a/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out
@@ -542,6 +542,22 @@ Location [not included in
comparison]/{warehouse_dir}/char_part
Partition Provider Catalog
+-- !query
+alter table char_part change column c1 comment 'char comment'
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+alter table char_part change column v1 comment 'varchar comment'
+-- !query schema
+struct<>
+-- !query output
+
+
+
-- !query
alter table char_part add partition (v2='ke', c2='nt') location 'loc1'
-- !query schema
@@ -555,8 +571,8 @@ desc formatted char_part
-- !query schema
struct<col_name:string,data_type:string,comment:string>
-- !query output
-c1 char(5)
-v1 varchar(6)
+c1 char(5) char comment
+v1 varchar(6) varchar comment
v2 varchar(2)
c2 char(2)
# Partition Information
@@ -598,8 +614,8 @@ desc formatted char_part
-- !query schema
struct<col_name:string,data_type:string,comment:string>
-- !query output
-c1 char(5)
-v1 varchar(6)
+c1 char(5) char comment
+v1 varchar(6) varchar comment
v2 varchar(2)
c2 char(2)
# Partition Information
@@ -633,8 +649,8 @@ desc formatted char_part
-- !query schema
struct<col_name:string,data_type:string,comment:string>
-- !query output
-c1 char(5)
-v1 varchar(6)
+c1 char(5) char comment
+v1 varchar(6) varchar comment
v2 varchar(2)
c2 char(2)
# Partition Information
@@ -668,8 +684,8 @@ desc formatted char_part
-- !query schema
struct<col_name:string,data_type:string,comment:string>
-- !query output
-c1 char(5)
-v1 varchar(6)
+c1 char(5) char comment
+v1 varchar(6) varchar comment
v2 varchar(2)
c2 char(2)
# Partition Information
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
