This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e23d69b0f1da [SPARK-48709][SQL] Fix varchar type resolution mismatch
for DataSourceV2 CTAS
e23d69b0f1da is described below
commit e23d69b0f1daa22bcd74a82c86aa93f66f4cba4d
Author: Yuming Wang <[email protected]>
AuthorDate: Wed Jun 26 17:41:41 2024 +0800
[SPARK-48709][SQL] Fix varchar type resolution mismatch for DataSourceV2
CTAS
### What changes were proposed in this pull request?
This PR fixes varchar type resolution mismatch for DataSourceV2 CTAS. For
example:
```sql
set spark.sql.storeAssignmentPolicy=LEGACY;
CREATE TABLE testcat.ns.t1 (d1 string, d2 varchar(200)) USING parquet;
CREATE TABLE testcat.ns.t2 USING foo as select * from testcat.ns.t1
```
Error message:
```
org.apache.spark.sql.AnalysisException: LEGACY store assignment policy is
disallowed in Spark data source V2. Please set the configuration
spark.sql.storeAssignmentPolicy to other values.
```
### Why are the changes needed?
Avoid query failures.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Unit test.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #47082 from wangyum/SPARK-48709.
Authored-by: Yuming Wang <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
---
.../apache/spark/sql/catalyst/plans/logical/v2Commands.scala | 3 ++-
.../org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala | 10 ++++++++++
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
index 48d48f0cd9e2..6339a18796fa 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
@@ -64,10 +64,11 @@ trait V2WriteCommand extends UnaryCommand with
KeepAnalyzedQuery with CTEInChild
table.skipSchemaResolution || (query.output.size == table.output.size &&
query.output.zip(table.output).forall {
case (inAttr, outAttr) =>
+ val inType =
CharVarcharUtils.getRawType(inAttr.metadata).getOrElse(inAttr.dataType)
val outType =
CharVarcharUtils.getRawType(outAttr.metadata).getOrElse(outAttr.dataType)
// names and types must match, nullability must be compatible
inAttr.name == outAttr.name &&
- DataType.equalsIgnoreCompatibleNullability(inAttr.dataType,
outType) &&
+ DataType.equalsIgnoreCompatibleNullability(inType, outType) &&
(outAttr.nullable || !inAttr.nullable)
})
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index 5b2472f77427..0382efaf9d7e 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -1739,6 +1739,16 @@ class DataSourceV2SQLSuiteV1Filter
}
}
+ test("SPARK-48709: varchar resolution mismatch for DataSourceV2 CTAS") {
+ withSQLConf(
+ SQLConf.STORE_ASSIGNMENT_POLICY.key ->
SQLConf.StoreAssignmentPolicy.LEGACY.toString) {
+ withTable("testcat.ns.t1", "testcat.ns.t2") {
+ sql("CREATE TABLE testcat.ns.t1 (d1 string, d2 varchar(200)) USING
parquet")
+ sql("CREATE TABLE testcat.ns.t2 USING foo as select * from
testcat.ns.t1")
+ }
+ }
+ }
+
test("ShowCurrentNamespace: basic tests") {
def testShowCurrentNamespace(expectedCatalogName: String,
expectedNamespace: String): Unit = {
val schema = new StructType()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
