c21 commented on a change in pull request #32495:
URL: https://github.com/apache/spark/pull/32495#discussion_r632283178
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
##########
@@ -418,115 +428,106 @@ case class SortMergeJoinExec(
// Inline mutable state since not many join operations in a task
val matches = ctx.addMutableState(clsName, "matches",
v => s"$v = new $clsName($inMemoryThreshold, $spillThreshold);",
forceInline = true)
- // Copy the left keys as class members so they could be used in next
function call.
- val matchedKeyVars = copyKeys(ctx, leftKeyVars)
+ // Copy the streamed keys as class members so they could be used in next
function call.
+ val matchedKeyVars = copyKeys(ctx, streamedKeyVars)
- ctx.addNewFunction("findNextInnerJoinRows",
+ ctx.addNewFunction("findNextJoinRows",
s"""
- |private boolean findNextInnerJoinRows(
- | scala.collection.Iterator leftIter,
- | scala.collection.Iterator rightIter) {
- | $leftRow = null;
+ |private boolean findNextJoinRows(
+ | scala.collection.Iterator streamedIter,
+ | scala.collection.Iterator bufferedIter) {
+ | $streamedRow = null;
| int comp = 0;
- | while ($leftRow == null) {
- | if (!leftIter.hasNext()) return false;
- | $leftRow = (InternalRow) leftIter.next();
- | ${leftKeyVars.map(_.code).mkString("\n")}
- | if ($leftAnyNull) {
- | $leftRow = null;
+ | while ($streamedRow == null) {
+ | if (!streamedIter.hasNext()) return false;
+ | $streamedRow = (InternalRow) streamedIter.next();
+ | ${streamedKeyVars.map(_.code).mkString("\n")}
+ | if ($streamedAnyNull) {
+ | $streamedRow = null;
| continue;
| }
| if (!$matches.isEmpty()) {
- | ${genComparison(ctx, leftKeyVars, matchedKeyVars)}
+ | ${genComparison(ctx, streamedKeyVars, matchedKeyVars)}
| if (comp == 0) {
| return true;
| }
| $matches.clear();
| }
|
| do {
- | if ($rightRow == null) {
- | if (!rightIter.hasNext()) {
+ | if ($bufferedRow == null) {
+ | if (!bufferedIter.hasNext()) {
| ${matchedKeyVars.map(_.code).mkString("\n")}
| return !$matches.isEmpty();
| }
- | $rightRow = (InternalRow) rightIter.next();
- | ${rightKeyTmpVars.map(_.code).mkString("\n")}
- | if ($rightAnyNull) {
- | $rightRow = null;
+ | $bufferedRow = (InternalRow) bufferedIter.next();
+ | ${bufferedKeyTmpVars.map(_.code).mkString("\n")}
+ | if ($bufferedAnyNull) {
+ | $bufferedRow = null;
| continue;
| }
- | ${rightKeyVars.map(_.code).mkString("\n")}
+ | ${bufferedKeyVars.map(_.code).mkString("\n")}
| }
- | ${genComparison(ctx, leftKeyVars, rightKeyVars)}
+ | ${genComparison(ctx, streamedKeyVars, bufferedKeyVars)}
| if (comp > 0) {
- | $rightRow = null;
+ | $bufferedRow = null;
| } else if (comp < 0) {
| if (!$matches.isEmpty()) {
| ${matchedKeyVars.map(_.code).mkString("\n")}
| return true;
| }
- | $leftRow = null;
+ | $streamedRow = null;
| } else {
- | $matches.add((UnsafeRow) $rightRow);
- | $rightRow = null;
+ | $matches.add((UnsafeRow) $bufferedRow);
+ | $bufferedRow = null;
| }
- | } while ($leftRow != null);
+ | } while ($streamedRow != null);
| }
| return false; // unreachable
|}
""".stripMargin, inlineToOuterClass = true)
Review comment:
@cloud-fan - thanks for calling it out. I would like to make it future
proof by giving it a fresh name. Let me do it in a followup PR. Actually I was
wondering the same question as you when implementing and spent some time to
figuring it out.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
