pan3793 commented on code in PR #58514:
URL: https://github.com/apache/spark/pull/58514#discussion_r3953759150
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala:
##########
@@ -838,11 +838,18 @@ case class EnsureRequirements(
joinType: JoinType,
keyOrdering: Ordering[InternalRowComparableWrapper]):
Seq[InternalRowComparableWrapper] = {
val merged = if
(SQLConf.get.getConf(SQLConf.V2_BUCKETING_PARTITION_FILTER_ENABLED)) {
+ // Rows only match within a key group, so a group no output row can come
from is dropped.
+ // Only equi-joins reach here (SMJ/SHJ over `ExtractEquiJoinKeys`), so
Cross matches on its
+ // keys like Inner, as `canReplicateLeftSide` already assumes. Which
side's rows survive
+ // follows `PushExtraPredicateThroughJoin`, plus LeftSingle.
joinType match {
- case Inner =>
+ // neither side keeps unmatched rows
+ case _: InnerLike | LeftSemi =>
mergeAndDedupPartitionKeys(leftPartitionKeys, rightPartitionKeys,
intersect = true)
- case LeftOuter => leftPartitionKeys.distinct
+ // every left row is kept or tested
+ case LeftOuter | LeftAnti | LeftSingle | ExistenceJoin(_) =>
leftPartitionKeys.distinct
Review Comment:
@peter-toth, I opened SPARK-59294
(https://github.com/apache/spark/pull/58569) for the idea
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
