LuciferYang commented on code in PR #51490:
URL: https://github.com/apache/spark/pull/51490#discussion_r2206915627
##########
sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala:
##########
@@ -4964,38 +4964,40 @@ class SQLQuerySuite extends QueryTest with
SharedSparkSession with AdaptiveSpark
}
test("SPARK-52686: Union should be resolved only if there are no
duplicates") {
- withTable("t1", "t2", "t3") {
- sql("CREATE TABLE t1 (col1 STRING, col2 STRING, col3 STRING)")
- sql("CREATE TABLE t2 (col1 STRING, col2 DOUBLE, col3 STRING)")
- sql("CREATE TABLE t3 (col1 STRING, col2 DOUBLE, a STRING, col3 STRING)")
-
- for (confValue <- Seq(false, true)) {
- withSQLConf(
- SQLConf.UNION_IS_RESOLVED_WHEN_DUPLICATES_PER_CHILD_RESOLVED.key ->
confValue.toString
- ) {
- val analyzedPlan = sql(
- """SELECT
- | *
- |FROM (
- | SELECT col1, col2, NULL AS a, col1 FROM t1
- | UNION
- | SELECT col1, col2, NULL AS a, col3 FROM t2
- | UNION
- | SELECT * FROM t3
- |)""".stripMargin
- ).queryExecution.analyzed
-
- val projectCount = analyzedPlan.collect {
- case project: Project => project
- }.size
-
- // When UNION_IS_RESOLVED_WHEN_DUPLICATES_PER_CHILD_RESOLVED is
disabled, we resolve
- // outer Union before deduplicating ExprIds in inner union. Because
of this we get an
- // additional unnecessary Project (see SPARK-52686).
- if (confValue) {
- assert(projectCount == 7)
- } else {
- assert(projectCount == 8)
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+ withTable("t1", "t2", "t3") {
+ sql("CREATE TABLE t1 (col1 STRING, col2 STRING, col3 STRING)")
+ sql("CREATE TABLE t2 (col1 STRING, col2 DOUBLE, col3 STRING)")
+ sql("CREATE TABLE t3 (col1 STRING, col2 DOUBLE, a STRING, col3
STRING)")
+
+ for (confValue <- Seq(false, true)) {
+ withSQLConf(
+ SQLConf.UNION_IS_RESOLVED_WHEN_DUPLICATES_PER_CHILD_RESOLVED.key
-> confValue.toString
+ ) {
+ val analyzedPlan = sql(
+ """SELECT
+ | *
+ |FROM (
+ | SELECT col1, col2, NULL AS a, col1 FROM t1
+ | UNION
+ | SELECT col1, col2, NULL AS a, col3 FROM t2
+ | UNION
+ | SELECT * FROM t3
+ |)""".stripMargin
+ ).queryExecution.analyzed
+
+ val projectCount = analyzedPlan.collect {
+ case project: Project => project
+ }.size
+
+ // When UNION_IS_RESOLVED_WHEN_DUPLICATES_PER_CHILD_RESOLVED is
disabled, we resolve
+ // outer Union before deduplicating ExprIds in inner union.
Because of this we get an
+ // additional unnecessary Project (see SPARK-52686).
+ if (confValue) {
Review Comment:
In ANSI mode:
- UNION_IS_RESOLVED_WHEN_DUPLICATES_PER_CHILD_RESOLVED = false,
projectCount = 8
- UNION_IS_RESOLVED_WHEN_DUPLICATES_PER_CHILD_RESOLVED = true, projectCount
= 7
In non-ANSI mode:
- UNION_IS_RESOLVED_WHEN_DUPLICATES_PER_CHILD_RESOLVED = false,
projectCount = 9
- UNION_IS_RESOLVED_WHEN_DUPLICATES_PER_CHILD_RESOLVED = true, projectCount
= 8
It seems that the current test case design is also applicable to the
non-ANSI mode?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
