Github user davies commented on a diff in the pull request:
https://github.com/apache/spark/pull/8383#discussion_r38589087
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeOuterJoin.scala
---
@@ -271,3 +298,224 @@ private class RightOuterIterator(
override def getRow: InternalRow = resultProj(joinedRow)
}
+
+private class SortMergeFullJoinScanner(
+ leftKeyGenerator: Projection,
+ rightKeyGenerator: Projection,
+ keyOrdering: Ordering[InternalRow],
+ leftIter: RowIterator,
+ numLeftRows: LongSQLMetric,
+ rightIter: RowIterator,
+ numRightRows: LongSQLMetric,
+ boundCondition: InternalRow => Boolean,
+ leftNullRow: InternalRow,
+ rightNullRow: InternalRow) {
+ private[this] val joinedRow: JoinedRow = new JoinedRow()
+ private[this] var leftRow: InternalRow = _
+ private[this] var leftRowKey: InternalRow = _
+ private[this] var rightRow: InternalRow = _
+ private[this] var rightRowKey: InternalRow = _
+
+ private[this] var leftIndex: Int = _
+ private[this] var rightIndex: Int = _
+ private[this] val leftMatches: ArrayBuffer[InternalRow] = new
ArrayBuffer[InternalRow]
+ private[this] val rightMatches: ArrayBuffer[InternalRow] = new
ArrayBuffer[InternalRow]
+
+ private[this] var notMatchBitSet: BitSet = _
+ private[this] val foundRightIndex: ArrayBuffer[Int] = new
ArrayBuffer[Int]
+
+ private[this] var leftAdvanced: Boolean = false
+ private[this] var rightAdvanced: Boolean = false
+
+ // --- Private methods
--------------------------------------------------------------------------
+
+ /**
+ * Advance the left iterator and compute the new row's join key.
+ * @return true if the left iterator returned a row and false otherwise.
+ */
+ private def advancedLeft(): Boolean = {
+ if (leftIter.advanceNext()) {
+ leftRow = leftIter.getRow
+ leftRowKey = leftKeyGenerator(leftRow)
+ numLeftRows += 1
+ true
+ } else {
+ leftRow = null
+ leftRowKey = null
+ false
+ }
+ }
+
+ /**
+ * Advance the right iterator and compute the new row's join key.
+ * @return true if the right iterator returned a row and false otherwise.
+ */
+ private def advancedRight(): Boolean = {
+ if (rightIter.advanceNext()) {
+ rightRow = rightIter.getRow
+ rightRowKey = rightKeyGenerator(rightRow)
+ numRightRows += 1
+ true
+ } else {
+ rightRow = null
+ rightRowKey = null
+ false
+ }
+ }
+
+ /**
+ * Consume rows from both iterators. Keep record matching rows in
buffers until their
+ * row keys are different.
+ */
+ private def findMatchingRows(matchingKey: InternalRow): Unit = {
+ leftMatches.clear()
+ rightMatches.clear()
+
+ do {
+ leftMatches += leftRow.copy()
+ leftAdvanced = advancedLeft()
+ } while (leftAdvanced && (keyOrdering.compare(leftRowKey, matchingKey)
== 0))
+
+ do {
+ rightMatches += rightRow.copy()
+ rightAdvanced = advancedRight()
+ } while (rightAdvanced && (keyOrdering.compare(matchingKey,
rightRowKey) == 0))
+
+ leftIndex = 0
+ rightIndex = 0
+ notMatchBitSet = new BitSet(rightMatches.size)
+ }
+
+ /**
+ * Scan for next matching rows in buffers of both sides.
+ * @return true if next row(s) are found and false otherwise.
+ */
+ private def scanNextInBuffered(): Boolean = {
+ if (leftIndex < leftMatches.size) {
+ var found = false
+ do {
+ found = boundCondition(joinedRow(leftMatches(leftIndex),
rightMatches(rightIndex)))
+ if (!found) {
+ if (!foundRightIndex.contains(rightIndex)) {
+ notMatchBitSet.set(rightIndex)
+ }
+ rightIndex += 1
+ }
+ } while (!found && rightIndex < rightMatches.size)
+
+ // No match can be found
+ // Output only left row
+ if (!found) {
+ joinedRow(leftMatches(leftIndex), rightNullRow)
--- End diff --
It's possible that we had find a match for left row, but not in current
try, for example left row is (a=1, b=2), right matches are [(c=1, d=2), (c=1,
d=3)], the join condition is a=c and b<=d, then the result should be [(1, 2, 1,
2)], but currently it will return [(1, 2, 1, 2), (1, 2, null, null)]
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
