Kimahriman commented on a change in pull request #32448:
URL: https://github.com/apache/spark/pull/32448#discussion_r640228110
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala
##########
@@ -707,32 +710,63 @@ class DataFrameSetOperationsSuite extends QueryTest with
SharedSparkSession {
val df2 = Seq((1, UnionClass1c(1, 2L, UnionClass4(2, 3L)))).toDF("id",
"a")
var unionDf = df1.unionByName(df2, true)
- checkAnswer(unionDf,
- Row(0, Row(0, 1, Row(null, 1, null, "2"))) ::
- Row(1, Row(1, 2, Row(2, null, 3L, null))) :: Nil)
assert(unionDf.schema.toDDL ==
"`id` INT,`a` STRUCT<`a`: INT, `b`: BIGINT, " +
- "`nested`: STRUCT<`A`: INT, `a`: INT, `b`: BIGINT, `c`: STRING>>")
+ "`nested`: STRUCT<`a`: INT, `c`: STRING, `A`: INT, `b`: BIGINT>>")
+ checkAnswer(unionDf,
+ Row(0, Row(0, 1, Row(1, "2", null, null))) ::
+ Row(1, Row(1, 2, Row(null, null, 2, 3L))) :: Nil)
unionDf = df2.unionByName(df1, true)
- checkAnswer(unionDf,
- Row(1, Row(1, 2, Row(2, null, 3L, null))) ::
- Row(0, Row(0, 1, Row(null, 1, null, "2"))) :: Nil)
assert(unionDf.schema.toDDL ==
"`id` INT,`a` STRUCT<`a`: INT, `b`: BIGINT, " +
- "`nested`: STRUCT<`A`: INT, `a`: INT, `b`: BIGINT, `c`: STRING>>")
+ "`nested`: STRUCT<`A`: INT, `b`: BIGINT, `a`: INT, `c`: STRING>>")
+ checkAnswer(unionDf,
+ Row(1, Row(1, 2, Row(2, 3L, null, null))) ::
+ Row(0, Row(0, 1, Row(null, null, 1, "2"))) :: Nil)
val df3 = Seq((2, UnionClass1b(2, 3L, UnionClass3(4, 5L)))).toDF("id",
"a")
unionDf = df2.unionByName(df3, true)
- checkAnswer(unionDf,
- Row(1, Row(1, 2, Row(2, null, 3L))) ::
- Row(2, Row(2, 3, Row(null, 4, 5L))) :: Nil)
assert(unionDf.schema.toDDL ==
"`id` INT,`a` STRUCT<`a`: INT, `b`: BIGINT, " +
- "`nested`: STRUCT<`A`: INT, `a`: INT, `b`: BIGINT>>")
+ "`nested`: STRUCT<`A`: INT, `b`: BIGINT, `a`: INT>>")
+ checkAnswer(unionDf,
+ Row(1, Row(1, 2, Row(2, 3L, null))) ::
+ Row(2, Row(2, 3, Row(null, 5L, 4))) :: Nil)
}
}
+ test("SPARK-32376: Make unionByName null-filling behavior work with struct
columns" +
+ " - case-insensitive cases") {
Review comment:
I removed this test since it's not really testing anything new, just
added it to understand what was happening
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
