gustavodemorais commented on code in PR #29937:
URL: https://github.com/apache/flink/pull/29937#discussion_r4589393042


##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/rank/AbstractTopNFunction.java:
##########
@@ -150,9 +152,21 @@ protected AbstractTopNFunction(
         this.generateUpdateBefore = generateUpdateBefore;
         this.inputRowType = inputRowType;
         this.outputRankNumber = outputRankNumber;
+        this.emitsChangelogPerRank = emitsChangelogPerRank(rankRange, 
outputRankNumber);
         this.sortKeySelector = sortKeySelector;
     }
 
+    /**
+     * Whether a Top-N emits its changes per rank instead of per row. This is 
the case when it
+     * outputs the rank number, or when it skips the first ranks, e.g. only 
ranks 2 to 3. The
+     * per-row algorithm cannot handle skipped ranks.
+     */
+    public static boolean emitsChangelogPerRank(RankRange rankRange, boolean 
outputRankNumber) {

Review Comment:
   Done, inverted to `emitsChangelogPerRow`. Kept the field as per-rank since 
the call sites check that first.



##########
flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeysTest.scala:
##########


Review Comment:
   Added `testGetUpsertKeysOnRowNumberRank`.



##########
flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeys.scala:
##########
@@ -112,8 +113,37 @@ class FlinkRelMdUpsertKeys private extends 
MetadataHandler[UpsertKeys] {
         val fmq = FlinkRelMetadataQuery.reuseOrCreate(mq)
         val inputUpsertKeys = fmq.getUpsertKeys(rel.getInput)
         val inputImmutableColumns = fmq.getImmutableColumns(rel.getInput)
-        val inputKeys = filterKeys(inputUpsertKeys, rel.partitionKey, 
inputImmutableColumns)
-        FlinkRelMdUniqueKeys.INSTANCE.getRankUniqueKeys(rel, inputKeys)
+        val inputUpsertKeysWithPartitionKey =
+          filterKeys(inputUpsertKeys, rel.partitionKey, inputImmutableColumns)
+        FlinkRelMdUniqueKeys.INSTANCE.getRankUniqueKeys(
+          rel,
+          inputUpsertKeysKeptByRank(rel, inputUpsertKeysWithPartitionKey))
+    }
+  }
+
+  /**
+   * A streaming Top-N emits its changes in one of two ways:
+   *   - Per row, when the rank number is not in the output and the Top-N 
includes rank 1, e.g. the
+   *     top 3. A row that enters gets an INSERT. A row that drops out gets a 
DELETE. Input upsert
+   *     keys such as `id` stay valid.
+   *   - Per rank, when the rank number is in the output or the Top-N skips 
the first ranks, e.g.
+   *     only ranks 2 to 3. Each change says which row now holds a rank. So 
the changes of one `id`
+   *     are spread over several ranks. Then `id` is an upsert key only if it 
is part of the
+   *     partition key. In that case each partition holds one row.
+   *
+   * The (partition key, rank number) key is added by 
[[FlinkRelMdUniqueKeys.getRankUniqueKeys]].
+   * Window Top-N is insert-only and keeps all input upsert keys.
+   */

Review Comment:
   Done, moved the `getRankUniqueKeys` note to the call site.



##########
flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeys.scala:
##########
@@ -112,8 +113,37 @@ class FlinkRelMdUpsertKeys private extends 
MetadataHandler[UpsertKeys] {
         val fmq = FlinkRelMetadataQuery.reuseOrCreate(mq)
         val inputUpsertKeys = fmq.getUpsertKeys(rel.getInput)
         val inputImmutableColumns = fmq.getImmutableColumns(rel.getInput)
-        val inputKeys = filterKeys(inputUpsertKeys, rel.partitionKey, 
inputImmutableColumns)
-        FlinkRelMdUniqueKeys.INSTANCE.getRankUniqueKeys(rel, inputKeys)
+        val inputUpsertKeysWithPartitionKey =
+          filterKeys(inputUpsertKeys, rel.partitionKey, inputImmutableColumns)
+        FlinkRelMdUniqueKeys.INSTANCE.getRankUniqueKeys(
+          rel,
+          inputUpsertKeysKeptByRank(rel, inputUpsertKeysWithPartitionKey))
+    }
+  }
+
+  /**
+   * A streaming Top-N emits its changes in one of two ways:
+   *   - Per row, when the rank number is not in the output and the Top-N 
includes rank 1, e.g. the
+   *     top 3. A row that enters gets an INSERT. A row that drops out gets a 
DELETE. Input upsert
+   *     keys such as `id` stay valid.
+   *   - Per rank, when the rank number is in the output or the Top-N skips 
the first ranks, e.g.
+   *     only ranks 2 to 3. Each change says which row now holds a rank. So 
the changes of one `id`
+   *     are spread over several ranks. Then `id` is an upsert key only if it 
is part of the
+   *     partition key. In that case each partition holds one row.
+   *
+   * The (partition key, rank number) key is added by 
[[FlinkRelMdUniqueKeys.getRankUniqueKeys]].
+   * Window Top-N is insert-only and keeps all input upsert keys.
+   */
+  private def inputUpsertKeysKeptByRank(
+      rel: Rank,
+      inputUpsertKeys: JSet[ImmutableBitSet]): JSet[ImmutableBitSet] = {
+    val emitsChangelogPerRank =
+      AbstractTopNFunction.emitsChangelogPerRank(rel.rankRange, 
rel.outputRankNumber)
+    rel match {
+      case _: StreamPhysicalWindowRank => inputUpsertKeys

Review Comment:
   Done



##########
flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeys.scala:
##########
@@ -112,8 +113,37 @@ class FlinkRelMdUpsertKeys private extends 
MetadataHandler[UpsertKeys] {
         val fmq = FlinkRelMetadataQuery.reuseOrCreate(mq)
         val inputUpsertKeys = fmq.getUpsertKeys(rel.getInput)
         val inputImmutableColumns = fmq.getImmutableColumns(rel.getInput)
-        val inputKeys = filterKeys(inputUpsertKeys, rel.partitionKey, 
inputImmutableColumns)
-        FlinkRelMdUniqueKeys.INSTANCE.getRankUniqueKeys(rel, inputKeys)
+        val inputUpsertKeysWithPartitionKey =
+          filterKeys(inputUpsertKeys, rel.partitionKey, inputImmutableColumns)
+        FlinkRelMdUniqueKeys.INSTANCE.getRankUniqueKeys(
+          rel,
+          inputUpsertKeysKeptByRank(rel, inputUpsertKeysWithPartitionKey))
+    }
+  }
+
+  /**
+   * A streaming Top-N emits its changes in one of two ways:
+   *   - Per row, when the rank number is not in the output and the Top-N 
includes rank 1, e.g. the
+   *     top 3. A row that enters gets an INSERT. A row that drops out gets a 
DELETE. Input upsert
+   *     keys such as `id` stay valid.
+   *   - Per rank, when the rank number is in the output or the Top-N skips 
the first ranks, e.g.
+   *     only ranks 2 to 3. Each change says which row now holds a rank. So 
the changes of one `id`
+   *     are spread over several ranks. Then `id` is an upsert key only if it 
is part of the
+   *     partition key. In that case each partition holds one row.
+   *
+   * The (partition key, rank number) key is added by 
[[FlinkRelMdUniqueKeys.getRankUniqueKeys]].
+   * Window Top-N is insert-only and keeps all input upsert keys.
+   */
+  private def inputUpsertKeysKeptByRank(
+      rel: Rank,
+      inputUpsertKeys: JSet[ImmutableBitSet]): JSet[ImmutableBitSet] = {
+    val emitsChangelogPerRank =
+      AbstractTopNFunction.emitsChangelogPerRank(rel.rankRange, 
rel.outputRankNumber)
+    rel match {
+      case _: StreamPhysicalWindowRank => inputUpsertKeys
+      case _ if inputUpsertKeys != null && emitsChangelogPerRank =>

Review Comment:
   Done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to