viirya commented on a change in pull request #23769: [SPARK-26864][SQL] Query
may return incorrect result when python udf is used as a join condition and the
udf uses attributes from both legs of left semi join.
URL: https://github.com/apache/spark/pull/23769#discussion_r256274697
##########
File path: python/pyspark/sql/tests/test_udf.py
##########
@@ -236,16 +225,6 @@ def test_udf_and_common_filter_in_join_condition(self):
# do not need spark.sql.crossJoin.enabled=true for udf is not the only
join condition.
self.assertEqual(df.collect(), [Row(a=1, a1=1, a2=1, b=1, b1=1, b2=1)])
- def test_udf_and_common_filter_in_left_semi_join_condition(self):
- # regression test for SPARK-25314
- # test the complex scenario with both udf and common filter
- left = self.spark.createDataFrame([Row(a=1, a1=1, a2=1), Row(a=2,
a1=2, a2=2)])
- right = self.spark.createDataFrame([Row(b=1, b1=1, b2=1), Row(b=1,
b1=3, b2=1)])
- f = udf(lambda a, b: a == b, BooleanType())
- df = left.join(right, [f("a", "b"), left.a1 == right.b1], "left_semi")
- # do not need spark.sql.crossJoin.enabled=true for udf is not the only
join condition.
- self.assertEqual(df.collect(), [Row(a=1, a1=1, a2=1)])
-
def test_udf_not_supported_in_join_condition(self):
# regression test for SPARK-25314
# test python udf is not supported in join type besides left_semi and
inner join.
Review comment:
Update this comment too?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
