Github user tdas commented on a diff in the pull request:
https://github.com/apache/spark/pull/19327#discussion_r140616618
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingSymmetricHashJoinExec.scala
---
@@ -216,22 +229,51 @@ case class StreamingSymmetricHashJoinExec(
}
// Filter the joined rows based on the given condition.
- val outputFilterFunction =
- newPredicate(condition.getOrElse(Literal(true)), left.output ++
right.output).eval _
- val filteredOutputIter =
- (leftOutputIter ++ rightOutputIter).filter(outputFilterFunction).map
{ row =>
- numOutputRows += 1
- row
- }
+ val outputFilterFunction =
newPredicate(condition.getOrElse(Literal(true)), output).eval _
+
+ val filteredInnerOutputIter = (leftOutputIter ++
rightOutputIter).filter(outputFilterFunction)
+
+ val outputIter: Iterator[InternalRow] = joinType match {
+ case Inner =>
+ filteredInnerOutputIter
+ case LeftOuter =>
+ val nullRight = new
GenericInternalRow(right.output.map(_.withNullability(true)).length)
+ filteredInnerOutputIter ++
+ leftSideJoiner
+ .removeOldState()
+ .filterNot { case (key, value) =>
rightSideJoiner.containsKey(key) }
+ .map { case (key, value) =>
joinedRow.withLeft(value).withRight(nullRight) }
+ case RightOuter =>
+ val nullLeft = new
GenericInternalRow(left.output.map(_.withNullability(true)).length)
+ filteredInnerOutputIter ++
+ rightSideJoiner
--- End diff --
nit: split this into two statements with an intermediate variable
`removedRowsIter`. and docs on what this does.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
