twalthr commented on a change in pull request #6776: [FLINK-9715][table]
Support temporal join with event time
URL: https://github.com/apache/flink/pull/6776#discussion_r222564861
##########
File path:
flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalRowtimeJoin.scala
##########
@@ -210,32 +215,46 @@ class TemporalRowtimeJoin(
if (rightRowIndex >= 0) {
val rightRow = rightRowsSorted.get(rightRowIndex)
- cRowWrapper.setChange(true)
- collector.setAbsoluteTimestamp(leftTime)
joinFunction.join(leftRow, rightRow, cRowWrapper)
}
leftIterator.remove()
}
else {
- lastUnprocessedTime = Some(
- Math.min(
- lastUnprocessedTime.getOrElse(Long.MaxValue),
- leftTime))
+ lastUnprocessedTime = Math.min(lastUnprocessedTime, leftTime)
}
}
- // remove all right entries older then the watermark, except the latest one
- // for example with rightState = [1, 5, 9] and watermark = 6
- // we can not remove "5" from rightState, because left elements with
rowtime of 7 or 8
- // could be joined with it later
- rightRowsSorted.map(rightRow => getRightTime(rightRow))
- .filter(rightTime => rightTime <= timerTimestamp)
- .dropRight(1)
- .foreach(rightKey => rightState.remove(rightKey))
-
+ cleanUpState(timerTimestamp, rightRowsSorted)
lastUnprocessedTime
}
+ /**
+ * Removes all right entries older then the watermark, except the latest
one. For example with:
+ * rightState = [1, 5, 9]
+ * and
+ * watermark = 6
+ * we can not remove "5" from rightState, because left elements with
rowtime of 7 or 8 could
+ * be joined with it later
+ */
+ private def cleanUpState(timerTimestamp: Long, rightRowsSorted:
util.List[Row]) = {
+ for (i <- 0 until firstIndexToKeep(timerTimestamp, rightRowsSorted)) {
+ val rightTime = getRightTime(rightRowsSorted.get(i))
+ rightState.remove(rightTime)
+ }
+ }
+
+ private def firstIndexToKeep(timerTimestamp: Long, rightRowsSorted:
util.List[Row]): Int = {
+ val firstIndexNewerThenTimer =
+ rightRowsSorted.indexWhere(row => getRightTime(row) > timerTimestamp)
Review comment:
You are using implicit Java<->Scala conversion here such that `indexWhere`
can be used. So my original concern does still exist.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
