wuchong commented on a change in pull request #11482: [FLINK-16581][table]
Minibatch deduplication lack state TTL bug fix
URL: https://github.com/apache/flink/pull/11482#discussion_r408713389
##########
File path:
flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamExecDeduplicate.scala
##########
@@ -117,21 +117,23 @@ class StreamExecDeduplicate(
val tableConfig = planner.getTableConfig
val isMiniBatchEnabled = tableConfig.getConfiguration.getBoolean(
ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_ENABLED)
+ val minRetentionTime = tableConfig.getMinIdleStateRetentionTime
+ val maxRetentionTime = tableConfig.getMaxIdleStateRetentionTime
val operator = if (isMiniBatchEnabled) {
val exeConfig = planner.getExecEnv.getConfig
val rowSerializer = rowTypeInfo.createSerializer(exeConfig)
val processFunction = if (keepLastRow) {
- new MiniBatchDeduplicateKeepLastRowFunction(rowTypeInfo,
generateRetraction, rowSerializer)
+ new MiniBatchDeduplicateKeepLastRowFunction(rowTypeInfo,
generateRetraction, rowSerializer,
+ minRetentionTime)
} else {
- new MiniBatchDeduplicateKeepFirstRowFunction(rowSerializer)
+ new MiniBatchDeduplicateKeepFirstRowFunction(rowSerializer,
+ minRetentionTime)
}
val trigger = AggregateUtil.createMiniBatchTrigger(tableConfig)
new KeyedMapBundleOperator(
processFunction,
trigger)
} else {
- val minRetentionTime = tableConfig.getMinIdleStateRetentionTime
- val maxRetentionTime = tableConfig.getMaxIdleStateRetentionTime
val processFunction = if (keepLastRow) {
new DeduplicateKeepLastRowFunction(minRetentionTime, maxRetentionTime,
rowTypeInfo,
Review comment:
Could you unify the behavior for non-mini-batch deduplication in this PR?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
