danielcweeks commented on code in PR #17552:
URL: https://github.com/apache/iceberg/pull/17552#discussion_r3938083248
##########
kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/channel/TestCoordinator.java:
##########
@@ -316,6 +320,212 @@ private void triggerCommitCycle(Coordinator coordinator) {
coordinator.process();
}
+ @Test
+ public void testCommitConsumerOffsetsDoesNotRewind() {
+ when(config.commitIntervalMs()).thenReturn(0);
+ when(config.commitTimeoutMs()).thenReturn(Integer.MAX_VALUE);
+
+ SinkTaskContext context = mock(SinkTaskContext.class);
+ Coordinator coordinator =
+ new Coordinator(catalog, config, ImmutableList.of(), clientFactory,
context);
+ coordinator.start();
+ initConsumer();
+
+ TopicPartition ctl = new TopicPartition(CTL_TOPIC_NAME, 0);
+
+ long healthyWatermark = 100L;
+ coordinator.controlTopicOffsets().put(0, healthyWatermark);
+ coordinator.commitConsumerOffsets();
+
+ coordinator.controlTopicOffsets().put(0, 5L);
+ coordinator.commitConsumerOffsets();
+
+ OffsetAndMetadata committedOffsetAndMetadata =
+ consumer.committed(ImmutableSet.of(ctl)).get(ctl);
+ long committed = committedOffsetAndMetadata == null ? 0L :
committedOffsetAndMetadata.offset();
+
+ assertThat(committed)
+ .as("commitConsumerOffsets should not rewind the shared -coord
consumer group offsets")
+ .isEqualTo(healthyWatermark);
+ }
+
+ @Test
+ public void testCommitConsumerDuplicateDoesNotCommit() {
+ when(config.commitIntervalMs()).thenReturn(0);
+ when(config.commitTimeoutMs()).thenReturn(Integer.MAX_VALUE);
+
+ SinkTaskContext context = mock(SinkTaskContext.class);
+ Coordinator coordinator =
+ new Coordinator(catalog, config, ImmutableList.of(), clientFactory,
context);
+ coordinator.start();
+ initConsumer();
+
+ TopicPartition ctl = new TopicPartition(CTL_TOPIC_NAME, 0);
+
+ long healthyWatermark = 100L;
+ coordinator.controlTopicOffsets().put(0, healthyWatermark);
+ coordinator.commitConsumerOffsets();
+
+ long nextWatermark = healthyWatermark + 5;
+ consumer.commitSync(ImmutableMap.of(ctl, new
OffsetAndMetadata(nextWatermark)));
+
+ coordinator.controlTopicOffsets().put(0, 100L);
+ coordinator.commitConsumerOffsets();
+
+ OffsetAndMetadata committedOffsetAndMetadata =
+ consumer.committed(ImmutableSet.of(ctl)).get(ctl);
+ long committed = committedOffsetAndMetadata == null ? 0L :
committedOffsetAndMetadata.offset();
+
+ assertThat(committed)
+ .as(
+ "commitConsumerOffsets should not commit offsets when offset has
not changed relative to local cache")
+ .isEqualTo(nextWatermark);
+ }
+
+ @Test
+ public void
testCommitConsumerRewindsOffsetsWhenAnotherCoordinatorAdvancesOffsets() {
+ when(config.commitIntervalMs()).thenReturn(0);
+ when(config.commitTimeoutMs()).thenReturn(Integer.MAX_VALUE);
+
+ SinkTaskContext context = mock(SinkTaskContext.class);
+ Coordinator coordinator =
+ new Coordinator(catalog, config, ImmutableList.of(), clientFactory,
context);
+ coordinator.start();
+ initConsumer();
+
+ TopicPartition ctl = new TopicPartition(CTL_TOPIC_NAME, 0);
+
+ long healthyWatermark = 100L;
+ coordinator.controlTopicOffsets().put(0, healthyWatermark);
+ coordinator.commitConsumerOffsets();
+
+ long nextWatermark = healthyWatermark + 5;
+ consumer.commitSync(ImmutableMap.of(ctl, new
OffsetAndMetadata(nextWatermark)));
+
+ OffsetAndMetadata anotherCommittedOffsetAndMetadata =
+ consumer.committed(ImmutableSet.of(ctl)).get(ctl);
+ long anotherCommitted =
+ anotherCommittedOffsetAndMetadata == null ? 0L :
anotherCommittedOffsetAndMetadata.offset();
+ assertThat(anotherCommitted)
+ .as("Precondition: another coordinator advanced the shared offset")
+ .isEqualTo(nextWatermark);
+
+ long rewindWatermark = healthyWatermark + 3;
+ coordinator.controlTopicOffsets().put(0, rewindWatermark);
+ coordinator.commitConsumerOffsets();
+
+ OffsetAndMetadata committedOffsetAndMetadata =
+ consumer.committed(ImmutableSet.of(ctl)).get(ctl);
+ long committed = committedOffsetAndMetadata == null ? 0L :
committedOffsetAndMetadata.offset();
+
+ assertThat(committed)
+ .as(
+ "Expected Limitation: commitConsumerOffsets will rewind offsets if
local offset advances and another coordinator committed a higher offset")
+ .isEqualTo(rewindWatermark)
+ .isLessThan(nextWatermark);
Review Comment:
This is not a great test. Generally we don't want to assert bad results.
Here, we're actually showing that there's a problem and flagging as "passed".
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
