rgyhuang commented on code in PR #56249:
URL: https://github.com/apache/spark/pull/56249#discussion_r3338379907
##########
sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala:
##########
@@ -1993,6 +1994,40 @@ class DataFrameFunctionsSuite extends SharedSparkSession
{
)
}
+ test("array_join with nullable nullReplacement under whole-stage codegen") {
+ // With a nullable nullReplacement column and an upstream IsNotNull
+ // filter that tightens the array (and delimiter) to non-nullable,
whole-stage codegen used to
+ // build the joined string but leave ev.isNull = true, discarding every
row as NULL. The result
+ // must match interpreted eval(). The source is materialized via a cached
temp view (an
+ // InMemoryRelation), so the plan is not folded to interpreted eval by
ConvertToLocalRelation.
+ withTempView("array_join_codegen") {
+ Seq(
+ (Seq[String]("a", null, "b"), ",", "NR"),
+ (Seq[String]("a", null, "b"), ",", null),
+ (Seq[String]("x", "y"), "-", "NR")
+ ).toDF("arr", "delim_col",
"repl_col").createOrReplaceTempView("array_join_codegen")
+ spark.catalog.cacheTable("array_join_codegen")
+
+ val query =
+ "SELECT array_join(arr, ',', repl_col) FROM array_join_codegen " +
Review Comment:
Thanks! Will apply delim_col suggestion and update checkAnswer rows.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
