Swinky commented on a change in pull request #31712:
URL: https://github.com/apache/spark/pull/31712#discussion_r589069701
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/subquery.scala
##########
@@ -98,55 +98,59 @@ object RewritePredicateSubquery extends Rule[LogicalPlan]
with PredicateHelper {
splitConjunctivePredicates(condition)
.partition(SubqueryExpression.hasInOrCorrelatedExistsSubquery)
- // Construct the pruned filter condition.
- val newFilter: LogicalPlan = withoutSubquery match {
- case Nil => child
- case conditions => Filter(conditions.reduce(And), child)
- }
-
- // Filter the plan by applying left semi and left anti joins.
- withSubquery.foldLeft(newFilter) {
- case (p, Exists(sub, conditions, _)) =>
- val (joinCond, outerPlan) = rewriteExistentialExpr(conditions, p)
- buildJoin(outerPlan, sub, LeftSemi, joinCond)
- case (p, Not(Exists(sub, conditions, _))) =>
- val (joinCond, outerPlan) = rewriteExistentialExpr(conditions, p)
- buildJoin(outerPlan, sub, LeftAnti, joinCond)
- case (p, InSubquery(values, ListQuery(sub, conditions, _, _))) =>
- // Deduplicate conflicting attributes if any.
- val newSub = dedupSubqueryOnSelfJoin(p, sub, Some(values))
- val inConditions = values.zip(newSub.output).map(EqualTo.tupled)
- val (joinCond, outerPlan) = rewriteExistentialExpr(inConditions ++
conditions, p)
- Join(outerPlan, newSub, LeftSemi, joinCond, JoinHint.NONE)
- case (p, Not(InSubquery(values, ListQuery(sub, conditions, _, _)))) =>
- // This is a NULL-aware (left) anti join (NAAJ) e.g. col NOT IN expr
- // Construct the condition. A NULL in one of the conditions is
regarded as a positive
- // result; such a row will be filtered out by the Anti-Join operator.
-
- // Note that will almost certainly be planned as a Broadcast Nested
Loop join.
- // Use EXISTS if performance matters to you.
+ withSubquery match {
Review comment:
done, thanks @allisonwang-db
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
