Efrat19 commented on code in PR #305:
URL:
https://github.com/apache/flink-connector-kafka/pull/305#discussion_r4657621582
##########
flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/KafkaSourceEnumerator.java:
##########
@@ -376,9 +390,26 @@ private void checkPartitionChanges(Set<TopicPartition>
fetchedPartitions, Throwa
if (partitionChange.isEmpty() && !boundedOneTimeDiscovery) {
return;
}
+ partitionInitializationInFlight = true;
context.callAsync(
- () -> initializePartitionSplits(partitionChange),
- this::handlePartitionSplitChanges);
+ () -> {
+ try {
+ return initializePartitionSplits(partitionChange);
+ } catch (Throwable failure) {
+ // A context may suppress a failure without calling
the handler below:
+ // StoppableKafkaEnumContextProxy does when the
cluster is inactive.
+ // Nothing of this batch was assigned, so release the
flag and let the
+ // next discovery cycle retry it.
+ context.runInCoordinatorThread(
+ () -> partitionInitializationInFlight = false);
+ throw failure;
+ }
+ },
+ (result, error) -> {
+ // Clear before handling so that a throwing handler cannot
leave the flag set.
+ partitionInitializationInFlight = false;
Review Comment:
Nit - set `partitionInitializationInFlight = false;` inside
`handlePartitionSplitChanges` to avoid the extra handler lambda?
##########
flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/KafkaSourceEnumerator.java:
##########
@@ -366,6 +375,11 @@ private void checkPartitionChanges(Set<TopicPartition>
fetchedPartitions, Throwa
"Failed to list subscribed topic partitions due to ", t);
}
+ if (partitionInitializationInFlight) {
+ LOG.debug("Skipping discovery result because split initialization
is in flight.");
Review Comment:
Also can we log the skipped changed partitions?
##########
flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/source/enumerator/KafkaSourceEnumeratorTest.java:
##########
@@ -635,6 +640,166 @@ public void testPartitionChangeChecking() throws
Throwable {
}
}
+ /** Verifies that in-flight partitions are not rediscovered as new on the
next cycle. */
+ @Test
+ public void testInFlightPartitionsNotRediscoveredAsNew() throws Throwable {
+ try (MockSplitEnumeratorContext<KafkaPartitionSplit> context =
+ new MockSplitEnumeratorContext<>(NUM_SUBTASKS);
+ KafkaSourceEnumerator enumerator =
+ createEnumerator(context,
ENABLE_PERIODIC_PARTITION_DISCOVERY)) {
+ enumerator.start();
+
+ // Register readers so partitions can be assigned.
+ registerReader(context, enumerator, READER0);
+ registerReader(context, enumerator, READER1);
+ registerReader(context, enumerator, READER2);
+
+ // Run discovery once and leave the worker callable pending to
simulate a slow
+ // worker.
+ context.runPeriodicCallable(PARTITION_DISCOVERY_CALLABLE_INDEX);
+
+ // The worker callable is now pending.
+ assertThat(context.getOneTimeCallables())
+ .as("initializePartitionSplits should be pending on the
worker")
+ .hasSize(1);
+
+ // Fire a second cycle before the first init completes.
+ context.runPeriodicCallable(PARTITION_DISCOVERY_CALLABLE_INDEX);
+
+ // The second cycle dispatches nothing because partitions are in
flight.
+ assertThat(context.getOneTimeCallables())
+ .as(
+ "No additional initializePartitionSplits should be
dispatched while partitions are in flight")
+ .hasSize(1);
+
+ // Run the pending init.
+ context.runNextOneTimeCallable();
+
+ // Verify a single assignment without duplicates.
+ List<SplitsAssignment<KafkaPartitionSplit>> assignments =
+ context.getSplitsAssignmentSequence();
+ assertThat(assignments).as("Exactly one assignment batch
expected").hasSize(1);
+
+ // Count assigned partitions across topics.
+ int totalAssignedPartitions =
+
assignments.get(0).assignment().values().stream().mapToInt(List::size).sum();
+ int expectedPartitions =
+
KafkaSourceTestEnv.getPartitionsForTopics(PRE_EXISTING_TOPICS).size();
+ assertThat(totalAssignedPartitions)
+ .as("Each partition should be assigned exactly once, not
duplicated")
+ .isEqualTo(expectedPartitions);
+ }
+ }
+
+ /** Verifies that a migrated split in flight is not rediscovered on the
next cycle. */
+ @Test
+ public void testMigratedSplitInFlightNotRediscovered() throws Throwable {
+ TopicPartition migratedTp = new TopicPartition(TOPIC2, 0);
+ try (MockSplitEnumeratorContext<KafkaPartitionSplit> context =
+ new MockSplitEnumeratorContext<>(NUM_SUBTASKS);
+ KafkaSourceEnumerator enumerator =
+ createEnumerator(
+ context,
+ 1L,
+ OffsetsInitializer.earliest(),
+ PRE_EXISTING_TOPICS,
+ Collections.emptySet(),
+ List.of(new KafkaPartitionSplit(migratedTp,
MIGRATED)),
+ false,
+ new Properties())) {
+ enumerator.start();
+
+ // Register readers so partitions can be assigned.
+ registerReader(context, enumerator, READER0);
+ registerReader(context, enumerator, READER1);
+ registerReader(context, enumerator, READER2);
+
+ // First cycle dispatches init for migrated and fresh partitions.
+ context.runPeriodicCallable(PARTITION_DISCOVERY_CALLABLE_INDEX);
+ assertThat(context.getOneTimeCallables())
+ .as("First discovery should dispatch init")
+ .hasSize(1);
+
+ // Second cycle before init completes dispatches nothing.
+ context.runPeriodicCallable(PARTITION_DISCOVERY_CALLABLE_INDEX);
+ assertThat(context.getOneTimeCallables())
+ .as("Migrated split in flight should not trigger a second
dispatch")
+ .hasSize(1);
+
+ // Run the pending init and verify the migrated split is assigned.
+ context.runNextOneTimeCallable();
+ assertThat(getAllAssignSplits(context, PRE_EXISTING_TOPICS))
+ .extracting(KafkaPartitionSplit::getTopicPartition)
+ .contains(migratedTp);
+ }
+ }
+
+ /**
+ * Verifies that a partition whose initialization failed benignly is
discovered again. The
+ * context swallows the failure like StoppableKafkaEnumContextProxy does
for an inactive
+ * cluster, so the enumerator never sees an error callback and must
release the in-flight state
+ * itself.
+ */
+ @Test
+ public void testPartitionsReleasedWhenInitializationFailureIsSuppressed()
throws Throwable {
Review Comment:
Nit: This test passes against main - it isn't necessarily related for the
pr, but still a nice to have IMO
I'd move to a separate commit maybe, wdyt?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]