psavalle commented on code in PR #58201:
URL: https://github.com/apache/spark/pull/58201#discussion_r3831285036
##########
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala:
##########
@@ -202,6 +202,68 @@ class FilterPushdownSuite extends PlanTest {
comparePlans(optimized, correctAnswer)
}
+ test("SPARK-39481: avoid evaluating scalar Python UDFs twice") {
+ val evalTypes = Seq(
+ "regular" -> PythonEvalType.SQL_BATCHED_UDF,
+ "Arrow-optimized" -> PythonEvalType.SQL_ARROW_BATCHED_UDF,
+ "Arrow element-wise" -> PythonEvalType.SQL_ARROW_ELEMENTWISE_UDF,
+ "scalar Pandas element-wise" ->
PythonEvalType.SQL_SCALAR_PANDAS_ELEMENTWISE_UDF,
+ "scalar Pandas iterator element-wise" ->
+ PythonEvalType.SQL_SCALAR_PANDAS_ITER_ELEMENTWISE_UDF,
+ "scalar Arrow element-wise" ->
PythonEvalType.SQL_SCALAR_ARROW_ELEMENTWISE_UDF,
+ "scalar Arrow iterator element-wise" ->
+ PythonEvalType.SQL_SCALAR_ARROW_ITER_ELEMENTWISE_UDF,
+ "scalar Pandas" -> PythonEvalType.SQL_SCALAR_PANDAS_UDF,
+ "scalar Pandas iterator" -> PythonEvalType.SQL_SCALAR_PANDAS_ITER_UDF,
+ "scalar Arrow" -> PythonEvalType.SQL_SCALAR_ARROW_UDF,
+ "scalar Arrow iterator" -> PythonEvalType.SQL_SCALAR_ARROW_ITER_UDF)
+
+ evalTypes.foreach { case (name, evalType) =>
+ withClue(s"$name Python UDF: ") {
+ val pythonUDF = PythonUDF(
+ "pythonUDF",
+ null,
+ BooleanType,
+ Seq(attrA),
+ evalType,
+ udfDeterministic = true)
+ val originalQuery = testRelation
+ .select(pythonUDF.as("result"))
+ .where($"result")
+ .analyze
+
+ comparePlans(Optimize.execute(originalQuery), originalQuery)
+ }
+ }
+ }
+
+ test("SPARK-39481: all Python function expressions are expensive") {
Review Comment:
Updated as well.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
