ferenc-csaky commented on code in PR #29038:
URL: https://github.com/apache/flink/pull/29038#discussion_r4643413250
##########
flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorAlignmentTest.java:
##########
@@ -116,6 +116,70 @@ void testWatermarkAlignmentWithIdleness() throws Exception
{
}
}
+ @Test
+ void testWatermarkAlignmentStatePrunedAfterSubtaskFailureAndReset() throws
Exception {
+ try (AutoCloseableRegistry closeableRegistry = new
AutoCloseableRegistry()) {
+ SourceCoordinator<?, ?> sourceCoordinator1 =
+ getAndStartNewSourceCoordinator(
+ new WatermarkAlignmentParams(1000L, "group1",
Long.MAX_VALUE),
+ closeableRegistry);
+
+ int subtask0 = 0;
+ int subtask1 = 1;
+
+ // baseline sanity: the group is constrained by the lowest
reported watermark
+ reportWatermarkEvent(sourceCoordinator1, subtask0, 100);
+ assertLatestWatermarkAlignmentEvent(subtask0, 1100);
+
+ reportWatermarkEvent(sourceCoordinator1, subtask1, 200);
+ assertLatestWatermarkAlignmentEvent(subtask0, 1100);
+ assertLatestWatermarkAlignmentEvent(subtask1, 1100);
+
+ // the subtask holding the group back fails and is reset
+ sourceCoordinator1.executionAttemptFailed(
+ subtask0, 0, new RuntimeException("Artificial failure for
subtask 0"));
+ sourceCoordinator1.subtaskReset(subtask0, 1L);
+ CoordinatorTestUtils.waitForCoordinatorToProcessActions(
+ sourceCoordinator1.getContext());
+
+ sourceCoordinator1.announceCombinedWatermark();
+
+ // CORRECT expectation: with the failed subtask's state pruned,
the group should now
+ // only be constrained by the surviving subtask (200 + 1000
drift). The failed subtask
+ // will re-report its watermark after restart anyway.
+ assertLatestWatermarkAlignmentEvent(subtask1, 1200);
+ }
+ }
+
+ @Test
+ void testSubtaskThatNeverReportedReceivesNoAlignmentEvents() throws
Exception {
+ try (AutoCloseableRegistry closeableRegistry = new
AutoCloseableRegistry()) {
+ SourceCoordinator<?, ?> sourceCoordinator1 =
+ getAndStartNewSourceCoordinator(
+ new WatermarkAlignmentParams(1000L, "group1",
Long.MAX_VALUE),
+ closeableRegistry);
+
+ int subtask0 = 0;
+ int subtask2 = 2;
+
+ // subtask0 reports; subtask2 is ready (gateway registered) but
never reports a
+ // watermark (e.g. idle from birth)
+ reportWatermarkEvent(sourceCoordinator1, subtask0, 100);
+ assertLatestWatermarkAlignmentEvent(subtask0, 1100);
+
+ sourceCoordinator1.announceCombinedWatermark();
+ sourceCoordinator1.announceCombinedWatermark();
+
+ // characterization: announceCombinedWatermark only iterates over
subtasks that have
+ // reported (combinedWatermark.keySet()), so a never-reporting
subtask never receives
+ // any WatermarkAlignmentEvent
+ List<OperatorEvent> eventsForSilentSubtask =
+ receivingTasks.getSentEventsForSubtask(subtask2);
+ assertThat(eventsForSilentSubtask)
+ .noneMatch(event -> event instanceof
WatermarkAlignmentEvent);
+ }
+ }
Review Comment:
Both cases need two sources sharing one `CoordinatorStore`, and
`getNewSourceCoordinator` gives each coordinator its own. On this branch the
two tests below fail with:
```
testResetOfLastReportingSubtaskDoesNotStallOtherSources:
expected: WatermarkAlignmentEvent{maxWatermark=1200}
but was: WatermarkAlignmentEvent{maxWatermark=-9223372036854774808}
testRecreatedCoordinatorDropsWatermarkOfPreviousOne:
expected: WatermarkAlignmentEvent{maxWatermark=1300}
but was: WatermarkAlignmentEvent{maxWatermark=1100}
```
```suggestion
}
@Test
void testResetOfLastReportingSubtaskDoesNotStallOtherSources() throws
Exception {
try (AutoCloseableRegistry closeableRegistry = new
AutoCloseableRegistry()) {
CoordinatorStoreImpl store = new CoordinatorStoreImpl();
SourceCoordinator<?, ?> sourceA =
getAndStartNewSourceCoordinator("sourceA", store,
closeableRegistry);
SourceCoordinator<?, ?> sourceB =
getAndStartNewSourceCoordinator("sourceB", store,
closeableRegistry);
reportWatermarkEvent(sourceA, 0, 100);
reportWatermarkEvent(sourceB, 1, 200);
assertLatestWatermarkAlignmentEvent(1, 1100);
// the only reporting subtask of sourceA is reset
sourceA.subtaskReset(0, 1L);
CoordinatorTestUtils.waitForCoordinatorToProcessActions(sourceA.getContext());
sourceB.announceCombinedWatermark();
assertLatestWatermarkAlignmentEvent(1, 1200);
}
}
@Test
void testRecreatedCoordinatorDropsWatermarkOfPreviousOne() throws
Exception {
try (AutoCloseableRegistry closeableRegistry = new
AutoCloseableRegistry()) {
CoordinatorStoreImpl store = new CoordinatorStoreImpl();
SourceCoordinator<?, ?> sourceA =
getAndStartNewSourceCoordinator("sourceA", store,
closeableRegistry);
SourceCoordinator<?, ?> sourceB =
getAndStartNewSourceCoordinator("sourceB", store,
closeableRegistry);
reportWatermarkEvent(sourceA, 0, 100);
reportWatermarkEvent(sourceB, 1, 200);
assertLatestWatermarkAlignmentEvent(1, 1100);
// global failover recreates sourceA's coordinator; its restored
readers never report
sourceA.close();
getAndStartNewSourceCoordinator("sourceA", store,
closeableRegistry);
reportWatermarkEvent(sourceB, 1, 300);
assertLatestWatermarkAlignmentEvent(1, 1300);
}
}
private SourceCoordinator<?, ?> getAndStartNewSourceCoordinator(
String operatorName,
CoordinatorStoreImpl coordinatorStore,
AutoCloseableRegistry closeableRegistry)
throws Exception {
SourceCoordinator<MockSourceSplit, Set<MockSourceSplit>>
sourceCoordinator =
new SourceCoordinator<>(
new JobID(),
operatorName,
createMockSource(),
getNewSourceCoordinatorContext(),
coordinatorStore,
new WatermarkAlignmentParams(1000L, "group1",
Long.MAX_VALUE),
null);
closeableRegistry.registerCloseable(sourceCoordinator);
sourceCoordinator.start();
setAllReaderTasksReady(sourceCoordinator);
return sourceCoordinator;
}
```
##########
flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinator.java:
##########
@@ -380,6 +380,13 @@ public void subtaskReset(int subtaskId, long checkpointId)
{
context.subtaskReset(subtaskId);
+ // Remove the last watermark reported by the failed
subtask so that a stale
+ // value does not keep constraining the watermark
alignment group. The
+ // restarted attempt re-registers itself with its next
ReportedWatermarkEvent.
+ combinedWatermark
+ .remove(subtaskId)
+ .ifPresent(this::updateAggregatedWatermarkOfGroup);
Review Comment:
If `subtaskId` is the last subtask of this source with a watermark,
`remove()` leaves the aggregator empty and `getAggregatedWatermark()` returns
`DEFAULT_WATERMARK` (`Long.MIN_VALUE`). That counts as a change, so it goes
into the group: every other source gets `Long.MIN_VALUE + drift` as its max
watermark until this source reports again, which never happens if its restored
readers stay idle. Before this PR they were capped at the stale value + drift.
It takes a parallelism-1 source, or a job that is a single pipelined region.
At startup a source with no reported watermark has no group entry, so I'd
drop the entry here too:
```suggestion
if (combinedWatermark.remove(subtaskId)) {
if (combinedWatermark.keySet().isEmpty()) {
removeFromGroup();
} else {
updateAggregatedWatermarkOfGroup(
combinedWatermark.getAggregatedWatermark());
}
}
```
Needs the `removeFromGroup()` and `remove()` suggestions below.
`testResetOfLastReportingSubtaskDoesNotStallOtherSources` covers it.
##########
flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinator.java:
##########
@@ -726,17 +733,18 @@ private void handleReportedWatermark(int subtask,
Watermark watermark) throws Fl
combinedWatermark
.aggregate(subtask, watermark)
- .ifPresent(
- newCombinedWatermark ->
- coordinatorStore.computeIfPresent(
-
watermarkAlignmentParams.getWatermarkGroup(),
- (key, oldValue) -> {
- WatermarkAggregator<String>
watermarkAggregator =
-
(WatermarkAggregator<String>) oldValue;
- watermarkAggregator.aggregate(
- operatorName,
newCombinedWatermark);
- return watermarkAggregator;
- }));
+ .ifPresent(this::updateAggregatedWatermarkOfGroup);
+ }
+
+ private void updateAggregatedWatermarkOfGroup(Watermark
newCombinedWatermark) {
+ coordinatorStore.computeIfPresent(
+ watermarkAlignmentParams.getWatermarkGroup(),
+ (key, oldValue) -> {
+ WatermarkAggregator<String> watermarkAggregator =
+ (WatermarkAggregator<String>) oldValue;
+ watermarkAggregator.aggregate(operatorName,
newCombinedWatermark);
+ return watermarkAggregator;
+ });
}
Review Comment:
A global failover (`DefaultScheduler#handleGlobalFailure`) recreates the
coordinator through `RecreateOnResetOperatorCoordinator` on the same
`ExecutionGraph`, so the `CoordinatorStore` keeps the old coordinator's group
entry. Only a report from a restored reader replaces it, and a reader whose
splits are all idle never reports (`SourceOperator#emitLatestWatermark` skips
`UNINITIALIZED`). It's the same stale entry this PR removes for region
failover, so I'd remove it in `close()` too:
```suggestion
}
private void removeFromGroup() {
coordinatorStore.computeIfPresent(
watermarkAlignmentParams.getWatermarkGroup(),
(key, oldValue) -> {
((WatermarkAggregator<String>)
oldValue).remove(operatorName);
return oldValue;
});
}
```
`close()` is outside the diff, so it can't take a suggestion. The call goes
after `closeQuietly(context)`, because closing the context stops the
coordinator thread, and a report still queued there could otherwise add the
entry back:
```java
closeQuietly(context);
if (watermarkAlignmentParams.isEnabled()) {
removeFromGroup();
}
```
`RecreateOnResetOperatorCoordinator` waits for the old `close()` (up to its
close timeout) before it creates the new coordinator, so this doesn't race with
the new one's reports. `testRecreatedCoordinatorDropsWatermarkOfPreviousOne`
covers it.
##########
flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinator.java:
##########
@@ -864,6 +872,27 @@ public Optional<Watermark> aggregate(T key, Watermark
watermark) {
return Optional.of(newAggregatedWatermark);
}
+ /**
+ * Removes the {@link Watermark} for the given {@code key}.
+ *
+ * @return the new updated combined {@link Watermark} if the value has
changed. {@code
+ * Optional.empty()} otherwise.
+ */
+ public Optional<Watermark> remove(T key) {
+ Watermark oldAggregatedWatermark = getAggregatedWatermark();
+
+ WatermarkElement removedWatermarkElement = watermarks.remove(key);
+ if (removedWatermarkElement != null) {
+ orderedWatermarks.remove(removedWatermarkElement);
+ }
+
+ Watermark newAggregatedWatermark = getAggregatedWatermark();
+ if (newAggregatedWatermark.equals(oldAggregatedWatermark)) {
+ return Optional.empty();
+ }
+ return Optional.of(newAggregatedWatermark);
+ }
Review Comment:
Once the caller handles the empty case, this only needs to say whether the
key was there. `Optional<Watermark>` can't tell "now empty" from a real new
minimum, which is how `DEFAULT_WATERMARK` got into the group.
```suggestion
* @return whether a {@link Watermark} was present for the given
{@code key}.
*/
public boolean remove(T key) {
WatermarkElement removedWatermarkElement =
watermarks.remove(key);
if (removedWatermarkElement == null) {
return false;
}
orderedWatermarks.remove(removedWatermarkElement);
return true;
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]