cloud-fan commented on code in PR #52537:
URL: https://github.com/apache/spark/pull/52537#discussion_r2415264427
##########
sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala:
##########
@@ -3144,4 +3144,106 @@ class ColumnExpressionSuite extends QueryTest with
SharedSparkSession {
checkAnswer(df.select($"dd" / ($"num" + 3)),
Seq((Duration.ofDays(2))).toDF())
}
+
+ test("Column.transform: basic transformation") {
+ val df = Seq(1, 2, 3).toDF("value")
+
+ def addOne(c: Column): Column = c + 1
+
+ checkAnswer(
+ df.select($"value".transform(addOne)),
+ Seq(2, 3, 4).toDF()
+ )
+ }
+
+ test("Column.transform: chaining transformations") {
+ val df = Seq("apple", "banana", "cherry").toDF("fruit")
+
+ def addPrefix(c: Column): Column = concat(lit("fruit_"), c)
+
+ def uppercase(c: Column): Column = upper(c)
+
+ checkAnswer(
+ df.select($"fruit".transform(addPrefix).transform(uppercase)),
+ Seq("FRUIT_APPLE", "FRUIT_BANANA", "FRUIT_CHERRY").toDF()
+ )
+ }
+
+ test("Column.transform: with complex function") {
+ val df = Seq(1, 2, 3, 4, 5).toDF("num")
+
+ def conditionalDouble(c: Column): Column = when(c > 3, c * 2).otherwise(c)
+
+ checkAnswer(
+ df.select($"num".transform(conditionalDouble)),
+ Seq(1, 2, 3, 8, 10).toDF()
+ )
+ }
+
+ test("Column.transform: with functions from functions object") {
+ val df = Seq(" hello ", " world ").toDF("text")
+
+ def trimAndUpper(c: Column): Column = upper(trim(c))
Review Comment:
This does not increase test coverage as it's the same as the basic function
with a named function. We should test something like
`col.transform(trim).transform(upper)`
##########
sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala:
##########
@@ -3144,4 +3144,106 @@ class ColumnExpressionSuite extends QueryTest with
SharedSparkSession {
checkAnswer(df.select($"dd" / ($"num" + 3)),
Seq((Duration.ofDays(2))).toDF())
}
+
+ test("Column.transform: basic transformation") {
+ val df = Seq(1, 2, 3).toDF("value")
+
+ def addOne(c: Column): Column = c + 1
+
+ checkAnswer(
+ df.select($"value".transform(addOne)),
+ Seq(2, 3, 4).toDF()
+ )
+ }
+
+ test("Column.transform: chaining transformations") {
+ val df = Seq("apple", "banana", "cherry").toDF("fruit")
+
+ def addPrefix(c: Column): Column = concat(lit("fruit_"), c)
+
+ def uppercase(c: Column): Column = upper(c)
+
+ checkAnswer(
+ df.select($"fruit".transform(addPrefix).transform(uppercase)),
+ Seq("FRUIT_APPLE", "FRUIT_BANANA", "FRUIT_CHERRY").toDF()
+ )
+ }
+
+ test("Column.transform: with complex function") {
+ val df = Seq(1, 2, 3, 4, 5).toDF("num")
+
+ def conditionalDouble(c: Column): Column = when(c > 3, c * 2).otherwise(c)
+
+ checkAnswer(
+ df.select($"num".transform(conditionalDouble)),
+ Seq(1, 2, 3, 8, 10).toDF()
+ )
+ }
+
+ test("Column.transform: with functions from functions object") {
+ val df = Seq(" hello ", " world ").toDF("text")
+
+ def trimAndUpper(c: Column): Column = upper(trim(c))
Review Comment:
This does not increase test coverage as it's the same as the basic test with
a named function. We should test something like
`col.transform(trim).transform(upper)`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
