This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new 3d7a20a08ec1 [SPARK-48709][SQL][3.5] Fix varchar type resolution
mismatch for DataSourceV2 CTAS
3d7a20a08ec1 is described below
commit 3d7a20a08ec104e98f15fa0926547086904a0fe8
Author: Yuming Wang <[email protected]>
AuthorDate: Fri Jun 28 00:12:13 2024 +0800
[SPARK-48709][SQL][3.5] Fix varchar type resolution mismatch for
DataSourceV2 CTAS
Backport of #47082.
### What changes were proposed in this pull request?
This PR fixes varchar type resolution mismatch for DataSourceV2 CTAS. For
example:
```sql
set spark.sql.storeAssignmentPolicy=LEGACY;
CREATE TABLE testcat.ns.t1 (d1 string, d2 varchar(200)) USING parquet;
CREATE TABLE testcat.ns.t2 USING foo as select * from testcat.ns.t1
```
Error message:
```
org.apache.spark.sql.AnalysisException: LEGACY store assignment policy is
disallowed in Spark data source V2. Please set the configuration
spark.sql.storeAssignmentPolicy to other values.
```
### Why are the changes needed?
Avoid query failures.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Unit test.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #47103 from wangyum/SPARK-48709-branch-3.5.
Authored-by: Yuming Wang <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
---
.../apache/spark/sql/catalyst/plans/logical/v2Commands.scala | 3 ++-
.../org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala | 10 ++++++++++
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
index 739ffa487e39..805f277cf9f6 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
@@ -62,10 +62,11 @@ trait V2WriteCommand extends UnaryCommand with
KeepAnalyzedQuery {
table.skipSchemaResolution || (query.output.size == table.output.size &&
query.output.zip(table.output).forall {
case (inAttr, outAttr) =>
+ val inType =
CharVarcharUtils.getRawType(inAttr.metadata).getOrElse(inAttr.dataType)
val outType =
CharVarcharUtils.getRawType(outAttr.metadata).getOrElse(outAttr.dataType)
// names and types must match, nullability must be compatible
inAttr.name == outAttr.name &&
- DataType.equalsIgnoreCompatibleNullability(inAttr.dataType,
outType) &&
+ DataType.equalsIgnoreCompatibleNullability(inType, outType) &&
(outAttr.nullable || !inAttr.nullable)
})
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index 6df71d08ef72..1db63f319728 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -1736,6 +1736,16 @@ class DataSourceV2SQLSuiteV1Filter
}
}
+ test("SPARK-48709: varchar resolution mismatch for DataSourceV2 CTAS") {
+ withSQLConf(
+ SQLConf.STORE_ASSIGNMENT_POLICY.key ->
SQLConf.StoreAssignmentPolicy.LEGACY.toString) {
+ withTable("testcat.ns.t1", "testcat.ns.t2") {
+ sql("CREATE TABLE testcat.ns.t1 (d1 string, d2 varchar(200)) USING
parquet")
+ sql("CREATE TABLE testcat.ns.t2 USING foo as select * from
testcat.ns.t1")
+ }
+ }
+ }
+
test("ShowCurrentNamespace: basic tests") {
def testShowCurrentNamespace(expectedCatalogName: String,
expectedNamespace: String): Unit = {
val schema = new StructType()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
