LukaZdravic commented on code in PR #59102:
URL: https://github.com/apache/spark/pull/59102#discussion_r4142688676
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeAsOfJoinExec.scala:
##########
@@ -424,10 +406,38 @@ private[joins] class SortMergeAsOfJoinScanner(
}
/**
- * Forward scan for Forward/Nearest joins: distance-based termination.
- * Stop when distance starts increasing past the minimum found so far.
+ * Forward joins: returns the first row that passes both conditions. The
buffer is sorted
+ * ascending, so that row is the closest and no distance is needed. The scan
stops once the
+ * as-of condition turns false after being true, since it stays false after
that.
*/
- private def findBestForwardNearest(leftRow: InternalRow): InternalRow = {
+ private def findFirstForward(leftRow: InternalRow): InternalRow = {
+ val iter = rightGroupBuffer.generateIterator()
+ var asOfSeen = false
+
+ joinedRow.withLeft(leftRow)
+ while (iter.hasNext) {
+ val rightRow = iter.next()
+ joinedRow.withRight(rightRow)
+
+ if (holds(boundAsOfCond)) {
+ asOfSeen = true
+ if (residualHolds) {
+ // No copy: the caller projects this row before the buffer iterator
moves again.
+ return rightRow
+ }
+ } else if (asOfSeen) {
+ // Past the tolerance bound, so no later row passes the as-of
condition.
+ return null
+ }
+ }
+ null
+ }
+
+ /**
+ * Nearest joins: keeps the row with the smallest distance. The scan stops
once the
+ * distance starts to grow past the minimum found so far.
Review Comment:
Done, thanks!
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeAsOfJoinExec.scala:
##########
@@ -189,7 +177,9 @@ private[joins] class SortMergeAsOfJoinScanner(
}
private val boundAsOfCond = bindReference(asOfCondition, joinedOutput)
- private val boundOrderExpr = bindReference(orderExpression, joinedOutput)
+ // Lazy, since only Nearest reads the distance. Its type can have no
ordering, for example
+ // CalendarInterval from TIMESTAMP - TIMESTAMP with
spark.sql.legacy.interval.enabled.
Review Comment:
Removed, thanks
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
