lianetm commented on code in PR #20521:
URL: https://github.com/apache/kafka/pull/20521#discussion_r2376925117
##########
clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java:
##########
@@ -217,35 +235,109 @@ public void process(ApplicationEvent event) {
}
}
- private void process(final PollEvent event) {
- // Trigger a reconciliation that can safely commit offsets if needed
to rebalance,
- // as we're processing before any new fetching starts in the app thread
-
requestManagers.consumerMembershipManager.ifPresent(consumerMembershipManager ->
- consumerMembershipManager.maybeReconcile(true));
- if (requestManagers.commitRequestManager.isPresent()) {
- CommitRequestManager commitRequestManager =
requestManagers.commitRequestManager.get();
- commitRequestManager.updateTimerAndMaybeCommit(event.pollTimeMs());
- // all commit request generation points have been passed,
- // so it's safe to notify the app thread could proceed and start
fetching
- event.markReconcileAndAutoCommitComplete();
- requestManagers.consumerHeartbeatRequestManager.ifPresent(hrm -> {
- hrm.membershipManager().onConsumerPoll();
- hrm.resetPollTimer(event.pollTimeMs());
- });
- requestManagers.streamsGroupHeartbeatRequestManager.ifPresent(hrm
-> {
- hrm.membershipManager().onConsumerPoll();
- hrm.resetPollTimer(event.pollTimeMs());
- });
- } else {
- // safe to unblock - no auto-commit risk here:
- // 1. commitRequestManager is not present
- // 2. shareConsumer has no auto-commit mechanism
- event.markReconcileAndAutoCommitComplete();
- requestManagers.shareHeartbeatRequestManager.ifPresent(hrm -> {
- hrm.membershipManager().onConsumerPoll();
- hrm.resetPollTimer(event.pollTimeMs());
+ private void process(final CompositePollEvent event) {
+ if (maybeFailCompositePoll(event) || maybePauseCompositePoll(event,
ApplicationEvent.Type.POLL))
+ return;
+
+ ApplicationEvent.Type nextEventType = event.nextEventType();
+
+ if (nextEventType == ApplicationEvent.Type.POLL) {
+ log.debug("Processing {} logic for {}", nextEventType, event);
+ processPollEvent(event.pollTimeMs());
+ nextEventType = ApplicationEvent.Type.UPDATE_SUBSCRIPTION_METADATA;
+
+ if (maybeFailCompositePoll(event) ||
maybePauseCompositePoll(event, nextEventType))
+ return;
+ }
+
+ if (nextEventType ==
ApplicationEvent.Type.UPDATE_SUBSCRIPTION_METADATA) {
+ log.debug("Processing {} logic for {}", nextEventType, event);
+ processUpdatePatternSubscriptionEvent();
+ nextEventType = ApplicationEvent.Type.CHECK_AND_UPDATE_POSITIONS;
+
+ if (maybeFailCompositePoll(event) ||
maybePauseCompositePoll(event, nextEventType))
+ return;
+ }
Review Comment:
This maybe we don't really need it? given that we only need to ensure we
re-eval regex vs metadata before sending topics to the broker, and we do have a
hook for that already (`onConsumerPoll`). So what about removing this all
together, and move just the `processUpdatePatternSubscriptionEvent` to
`onConsumerPoll`
https://github.com/apache/kafka/blob/563338c0e950209edebf064353a626249c5efc3c/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractMembershipManager.java#L489
(renamed probably, it's not about events really, it's more of a
~`maybeUpdatePatternSubscription`)
##########
clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompositePollEventInvoker.java:
##########
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer.internals;
+
+import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent;
+import
org.apache.kafka.clients.consumer.internals.events.ApplicationEventHandler;
+import org.apache.kafka.clients.consumer.internals.events.CompositePollEvent;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Timer;
+
+import org.slf4j.Logger;
+
+import static
org.apache.kafka.clients.consumer.internals.events.CompletableEvent.calculateDeadlineMs;
+
+public class CompositePollEventInvoker {
+
+ private final Logger log;
+ private final Time time;
+ private final ApplicationEventHandler applicationEventHandler;
+ private final Runnable backgroundEventProcessor;
+ private final Runnable offsetCommitProcessor;
+ private CompositePollEvent inflight;
+
+ public CompositePollEventInvoker(LogContext logContext,
+ Time time,
+ ApplicationEventHandler
applicationEventHandler,
+ Runnable backgroundEventProcessor,
+ Runnable offsetCommitProcessor) {
+ this.log = logContext.logger(getClass());
+ this.time = time;
+ this.applicationEventHandler = applicationEventHandler;
+ this.backgroundEventProcessor = backgroundEventProcessor;
+ this.offsetCommitProcessor = offsetCommitProcessor;
+ }
+
+ public void poll(Timer timer) {
+ if (inflight == null) {
+ log.debug("No existing inflight event, submitting");
+ submitEvent(ApplicationEvent.Type.POLL, timer);
+ }
+
+ try {
+ if (log.isTraceEnabled()) {
+ log.trace(
+ "Attempting to retrieve result from previously submitted
{} with {} remaining on timer",
+ inflight,
+ timer.remainingMs()
+ );
+ }
+
+ CompositePollEvent.Result result = inflight.resultOrError();
+ CompositePollEvent.State state = result.state();
+
+ if (state == CompositePollEvent.State.COMPLETE) {
+ // Make sure to clear out the inflight request since it's
complete.
+ log.debug("Event {} completed, clearing inflight", inflight);
+ inflight = null;
+ } else if (state ==
CompositePollEvent.State.BACKGROUND_EVENT_PROCESSING_REQUIRED) {
+ log.debug("About to process background events");
+ backgroundEventProcessor.run();
+ log.debug("Done processing background events");
+ result.nextEventType().ifPresent(t -> submitEvent(t, timer));
+ } else if (state ==
CompositePollEvent.State.OFFSET_COMMIT_CALLBACKS_REQUIRED) {
+ log.debug("About to process offset commits");
+ offsetCommitProcessor.run();
+ log.debug("Done processing offset commits");
+ result.nextEventType().ifPresent(t -> submitEvent(t, timer));
Review Comment:
this bit is also one I wonder if needed, or if we can just trigger the
commit callbacks direclty from the app thread poll loop
(`offsetCommitCallbackInvoker.executeCallbacks()`) . If that works, no need for
this extra state or all the related `offsetCommitProcessor` logic
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
