hemanthboyina commented on code in PR #59161:
URL: https://github.com/apache/spark/pull/59161#discussion_r4210464613
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/planmerging/PlanMerger.scala:
##########
@@ -690,19 +684,43 @@ class PlanMerger(
// rows are NULL-padded so f=NULL, causing FILTER
(WHERE f) to incorrectly
// exclude rows that should contribute to the
aggregate. Right-side
// attributes are also absent from semi/anti join
output.
- (leftNPFilter.isEmpty && leftCPFilter.isEmpty ||
+ (leftNPFilter.isEmpty && leftCPFilter.isEmpty ||
filterSafeForJoin(fromLeft = true, cp.joinType)) &&
(rightNPFilter.isEmpty && rightCPFilter.isEmpty ||
filterSafeForJoin(fromLeft = false, cp.joinType)) =>
val npMapping = leftNPMapping ++ rightNPMapping
val mappedNPCondition = np.condition.map(mapAttributes(_,
npMapping))
// Comparing the canonicalized form is required to ignore
different forms of the
// same expression and `AttributeReference.qualifier`s in
`cp.condition`.
- if (mappedNPCondition.map(_.canonicalized) ==
cp.condition.map(_.canonicalized)) {
- val npFilter = leftNPFilter.orElse(rightNPFilter)
- val cpFilter = leftCPFilter.orElse(rightCPFilter)
- Some(TryMergeResult(cp.withNewChildren(Seq(mergedLeft,
mergedRight)), npMapping,
- npFilter, cpFilter, dsv2Merged = leftDsv2Merged ||
rightDsv2Merged))
+ if (mappedNPCondition.map(_.canonicalized) ==
+ cp.condition.map(_.canonicalized)) {
+ val mergedJoin = cp.withNewChildren(Seq(mergedLeft,
mergedRight))
+ val (npFilter, npAlias) = (leftNPFilter, rightNPFilter)
match {
+ case (Some((left, _)), Some((right, _))) =>
+ val alias = Alias(
+ And(left, right),
s"propagatedFilter_${PlanMerger.newId}")()
Review Comment:
thanks, you are right. I reproduced the three sub query case locally and it
failed. Fixed it now
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
