gnodet-bot commented on code in PR #29992:
URL: https://github.com/apache/camel/pull/29992#discussion_r4576372762


##########
components/camel-kafka/src/test/java/org/apache/camel/component/kafka/dst/KafkaWorld.java:
##########
@@ -0,0 +1,557 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.kafka.dst;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.camel.Exchange;
+import org.apache.camel.Message;
+import org.apache.camel.ServiceStatus;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.kafka.KafkaClientFactory;
+import org.apache.camel.component.kafka.KafkaComponent;
+import org.apache.camel.component.kafka.KafkaConfiguration;
+import org.apache.camel.component.kafka.KafkaConstants;
+import 
org.apache.camel.component.kafka.consumer.DefaultKafkaManualAsyncCommitFactory;
+import 
org.apache.camel.component.kafka.consumer.DefaultKafkaManualCommitFactory;
+import org.apache.camel.component.kafka.consumer.KafkaManualCommit;
+import org.apache.camel.impl.DefaultCamelContext;
+import org.apache.camel.test.dst.Dst;
+import org.apache.camel.test.dst.DstCamel;
+import org.apache.camel.test.dst.DstExecutor;
+import org.apache.camel.test.dst.DstLedger;
+import org.apache.camel.test.dst.DstWorld;
+import org.apache.camel.test.dst.Invariants;
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.producer.Producer;
+
+/**
+ * The world around a camel-kafka consumer: a fake Kafka cluster and consumer 
group ({@link FakeKafka}), a producer, a
+ * route whose processing is sometimes slow and sometimes fails, and an 
operator that restarts the route, suspends and
+ * resumes it (with a second suspend and resume while the consumer still 
handles the first), and lets a phantom member
+ * join and leave the group, so that partitions are revoked and assigned again 
while records are processed. Faults: the
+ * consumer cannot be created, polls fail, commits fail or time out, commits 
are slow, processing fails.
+ * <p>
+ * The seed picks the commit mode: Kafka's auto-commit 
(autoCommitEnable=true), manual synchronous or asynchronous
+ * commits in the route (allowManualCommit=true with 
DefaultKafkaManualCommitFactory or
+ * DefaultKafkaManualAsyncCommitFactory, after every record or only for the 
last record of a poll), or batching
+ * (batching=true with autoCommitEnable=false: Camel commits each batch once 
it was processed), and breakOnFirstError
+ * and pollOnError (ERROR_HANDLER, DISCARD, RETRY, RECONNECT; not STOP, which 
ends the consumer by design).
+ * <p>
+ * Not explored: batching with Kafka's auto-commit (the auto-commit commits 
the position of records that Camel still
+ * holds in an incomplete batch, which the KafkaConsumer Javadoc warns about), 
and batching with breakOnFirstError (the
+ * failed batch is committed, see "Break on First Error in Batching Mode" in 
the component documentation); manual
+ * commits from another thread; consumersCount above 1; seekTo, offset 
repositories and resume strategies.
+ * <p>
+ * Left out of the default set: breakOnFirstError with the synchronous manual 
commits (SyncCommitManager), where a
+ * commitSync that times out in the break path is being looked at separately;
+ * {@code -Ddst.kafka.syncBreakOnFirstError=true} puts the combination back.
+ * <p>
+ * consumersCount is 1: a consumer thread loops for ever, and with the 
one-stack model a second looping thread nested
+ * above the first would never let it resume. For the same reason a route stop 
that runs while the consumer thread waits
+ * in poll cannot see it finish: the stop wakes up the Kafka consumer, waits 
for its timeouts, and the thread ends after
+ * the stop returned, so the operator starts the route again only once the 
previous Kafka consumer is closed. The
+ * operator does not stop or suspend the route while an exchange is in the 
route (the shutdown strategy would wait for
+ * it for ever, as it cannot see it finish) or while the consumer thread waits 
for the background task that creates its
+ * Kafka consumer (a stop interrupts that wait with real threads, which the 
one-stack model does not do). The lock of
+ * KafkaFetchRecords does not exclude on the one stack, so a stop never waits 
for the processing of the current records.
+ * <p>
+ * Properties: a committed offset never passes a record the route has not 
finished with (at-least-once: completed, or
+ * failed with breakOnFirstError=false, which skips it by design); a suspended 
route stops fetching once its consumer
+ * handled the suspend; once the faults cleared, every record is finished, the 
committed offsets reach the end of each
+ * partition, and the route holds one open Kafka consumer; the consumer uses 
the client as the Javadoc requires; no
+ * Kafka consumer is left open after the route stopped. Records may be 
consumed more than once.
+ */
+@DstWorld(component = "kafka",
+          fake = "FakeKafka: a topic with one or two partitions and a consumer 
group (classic protocol, eager"
+                 + " rebalance), behind the Consumer interface of 
KafkaClientFactory",
+          covers = {
+                  "commits by Kafka's auto-commit, by manual synchronous or 
asynchronous commits in the route, and per"
+                     + " batch with batching=true",
+                  "breakOnFirstError",
+                  "pollOnError ERROR_HANDLER, DISCARD, RETRY and RECONNECT",
+                  "a poll that fails while a batch is incomplete",
+                  "route restart, also while the consumer polls",
+                  "route suspend and resume, with a second request while the 
consumer handles the first",
+                  "partitions revoked and assigned again by a rebalance while 
records are processed" },
+          faults = {
+                  "the Kafka consumer cannot be created", "poll fails", 
"commitSync times out",
+                  "commitAsync fails", "auto-commit fails", "commits are 
slow", "committed answers slowly",
+                  "processing fails", "processing is slow", "a member joins 
and leaves the group" },
+          invariants = {
+                  "the consumer uses the Kafka consumer as documented",
+                  "a committed offset never passes a record the route has not 
finished with",
+                  "a suspended route stops fetching",
+                  "every record is finished once the faults cleared",
+                  "the committed offsets reach the end once the faults 
cleared",
+                  "the route holds one Kafka consumer once the faults cleared",
+                  "no Kafka consumer is left open after the route stopped" },
+          issues = { "CAMEL-25313", "CAMEL-25756" })
+final class KafkaWorld {
+
+    static final String ROUTE = "kafka";
+
+    private final Dst dst;
+    private final DstLedger ledger;
+    private final FakeKafka kafka;
+    private final int partitions;
+    private final String mode;
+    private final boolean breakOnFirstError;
+    private final boolean commitEveryRecord;
+    private final String pollOnError;
+    private final long pollTimeoutMs;
+    private final long shutdownTimeout;
+    private final long commitTimeoutMs;
+    private final long backoffInterval;
+    private final double processFailRate;
+    private final List<String> violations = new ArrayList<>();
+    private boolean faults = true;
+    private int sendsLeft;
+    private int opsLeft;
+    // exchanges in the route: a route stop or suspend nested above one on the 
one stack would wait for it for ever
+    // (the shutdown strategy cannot see it finish), so the operator stops and 
suspends only when none is in the route
+    private int inRoute;
+    private boolean finished;
+    private String finalStatus;
+    private String finalOpen;
+    private int finalOpenCount;
+    private boolean finalDone;
+    private boolean finalCommitted;
+
+    KafkaWorld(Dst dst) {
+        this.dst = dst;
+        this.ledger = new DstLedger(dst);
+        this.partitions = dst.pick(List.of(1, 2));
+        this.kafka = new FakeKafka(dst, ledger, partitions);
+        this.mode = 
dst.pick(Arrays.asList(System.getProperty("dst.kafka.modes", 
"AUTO,SYNC,ASYNC,BATCH").split(",")));
+        // see the class Javadoc: batching with breakOnFirstError is not 
explored, and the synchronous manual commits
+        // with breakOnFirstError are left out unless 
-Ddst.kafka.syncBreakOnFirstError=true
+        this.breakOnFirstError = !"BATCH".equals(mode)
+                && (!"SYNC".equals(mode) || 
Boolean.getBoolean("dst.kafka.syncBreakOnFirstError"))
+                && dst.chance(0.5);
+        this.commitEveryRecord = dst.chance(0.5);
+        this.pollOnError = dst.pick(Arrays.asList(
+                System.getProperty("dst.kafka.pollOnError", 
"ERROR_HANDLER,DISCARD,RETRY,RECONNECT").split(",")));
+        this.pollTimeoutMs = dst.pick(List.of(100L, 1000L, 5000L));
+        this.shutdownTimeout = dst.pick(List.of(1000L, 30000L));
+        this.commitTimeoutMs = dst.pick(List.of(1000L, 5000L));
+        this.backoffInterval = dst.pick(List.of(500L, 5000L));
+        kafka.autoCommit = "AUTO".equals(mode);
+        kafka.autoCommitIntervalMs = dst.pick(List.of(1000L, 5000L));
+        kafka.breakOnFirstError = breakOnFirstError;
+        kafka.maxPollRecords = "BATCH".equals(mode) ? dst.pick(List.of(3, 10)) 
: dst.pick(List.of(1, 3, 10));
+        kafka.createFailRate = dst.pick(List.of(0.0, 0.1, 0.3));
+        kafka.pollFailRate = dst.pick(List.of(0.0, 0.02, 0.1));
+        kafka.commitFailRate = dst.pick(List.of(0.0, 0.05, 0.2));
+        kafka.slowCommits = dst.chance(0.5);
+        this.processFailRate = dst.pick(List.of(0.0, 0.05, 0.2));
+        dst.trace("world: " + config());
+    }
+
+    private String config() {
+        return "partitions=" + partitions + " mode=" + mode + " 
breakOnFirstError=" + breakOnFirstError
+               + " commitEveryRecord=" + commitEveryRecord + " pollOnError=" + 
pollOnError + " pollTimeoutMs="
+               + pollTimeoutMs + " maxPollRecords=" + kafka.maxPollRecords + " 
shutdownTimeout=" + shutdownTimeout
+               + " commitTimeoutMs=" + commitTimeoutMs + " 
autoCommitIntervalMs=" + kafka.autoCommitIntervalMs
+               + " backoff=" + backoffInterval + " createFail=" + 
kafka.createFailRate + " pollFail="
+               + kafka.pollFailRate + " commitFail=" + kafka.commitFailRate + 
" slowCommits=" + kafka.slowCommits
+               + " processFail=" + processFailRate;
+    }
+
+    void run() throws Exception {
+        DefaultCamelContext context = DstCamel.newContext(dst);
+        KafkaComponent component = new KafkaComponent();
+        component.setCreateConsumerBackoffInterval(backoffInterval);
+        if ("SYNC".equals(mode)) {
+            component.setKafkaManualCommitFactory(new 
DefaultKafkaManualCommitFactory());
+        } else if ("ASYNC".equals(mode)) {
+            component.setKafkaManualCommitFactory(new 
DefaultKafkaManualAsyncCommitFactory());
+        }
+        component.setKafkaClientFactory(new KafkaClientFactory() {
+            @Override
+            public Producer getProducer(Properties kafkaProps) {
+                throw new UnsupportedOperationException("the world has no 
producer endpoint");
+            }
+
+            @Override
+            public Consumer getConsumer(Properties kafkaProps) {
+                return kafka.newConsumer();
+            }
+
+            @Override
+            public String getBrokers(KafkaConfiguration configuration) {
+                return configuration.getBrokers();
+            }
+        });
+        context.addComponent("kafka", component);
+        boolean manual = "SYNC".equals(mode) || "ASYNC".equals(mode);
+        String uri = "kafka:" + FakeKafka.TOPIC + 
"?brokers=127.0.0.1:9092&groupId=" + FakeKafka.GROUP
+                     + 
"&consumersCount=1&autoOffsetReset=earliest&pollTimeoutMs=" + pollTimeoutMs + 
"&maxPollRecords="
+                     + kafka.maxPollRecords + "&shutdownTimeout=" + 
shutdownTimeout + "&commitTimeoutMs="
+                     + commitTimeoutMs + "&pollOnError=" + pollOnError + 
"&breakOnFirstError=" + breakOnFirstError
+                     + "&autoCommitEnable=" + "AUTO".equals(mode) + 
"&autoCommitIntervalMs="
+                     + kafka.autoCommitIntervalMs + "&allowManualCommit=" + 
manual + "&batching=" + "BATCH".equals(mode);
+        context.addRoutes(new RouteBuilder() {
+            @Override
+            public void configure() {
+                from(uri).routeId(ROUTE).autoStartup(false)
+                        .process(exchange -> process(exchange, manual));
+            }
+        });
+        context.start();
+        DstExecutor operator = dst.executor("operator", 1);
+        try {
+            operator.schedule("start route", 0, () -> startRoute(context, 
operator));
+            planSends();
+            planOperations(context, operator);
+            operator.schedule("finisher", 1000, () -> finisher(context, 
operator, -1));
+            dst.runUntil(() -> finished, "the finisher");
+            // the consumer thread ends after the stop; let it close its Kafka 
consumer
+            dst.runFor(2 * shutdownTimeout + 10_000);
+            dst.check(
+                    Invariants.of("the consumer uses the Kafka consumer as 
documented",
+                            () -> kafka.violations.isEmpty() && 
violations.isEmpty()
+                                    ? null : kafka.violations + " " + 
violations + " (" + config() + ")"),
+                    Invariants.of("a committed offset never passes a record 
the route has not finished with",
+                            () -> kafka.lossViolations.isEmpty() ? null : 
kafka.lossViolations + " (" + config() + ")"),
+                    Invariants.of("a suspended route stops fetching",
+                            () -> kafka.suspendViolations.isEmpty()
+                                    ? null : kafka.suspendViolations + " (" + 
config() + ")"),
+                    Invariants.of("every record is finished once the faults 
cleared",
+                            () -> finalDone ? null : "not finished: " + 
notDone() + " (" + config() + ")"),
+                    Invariants.of("the committed offsets reach the end once 
the faults cleared",
+                            () -> finalCommitted ? null : kafka.notCommitted() 
+ " (" + config() + ")"),
+                    Invariants.of("the route holds one Kafka consumer once the 
faults cleared",
+                            () -> "Started".equals(finalStatus) && 
finalOpenCount == 1
+                                    ? null
+                                    : "route is " + finalStatus + ", open 
Kafka consumers " + finalOpen + " ("
+                                      + config() + ")"),
+                    Invariants.of("no Kafka consumer is left open after the 
route stopped",
+                            () -> kafka.open.isEmpty()
+                                    ? null : "open after stop: " + 
kafka.openNames() + " (" + config() + ")"));
+        } finally {
+            dst.dropPending();
+            context.stop();
+        }
+    }
+
+    /** The route: a record (or each record of a batch) is routed, maybe 
slowly, then fails or completes. */
+    private void process(Exchange exchange, boolean manual) throws Exception {
+        List<FakeKafka.Rec> recs = new ArrayList<>();
+        Object body = exchange.getMessage().getBody();
+        if (!(body instanceof List) && 
exchange.getMessage().getHeader(KafkaConstants.OFFSET) == null) {
+            // not a record: an exception of the consumer bridged to the error 
handler
+            return;
+        }
+        if (body instanceof List<?> batch) {
+            for (Object o : batch) {
+                recs.add(recOf(((Exchange) o).getMessage()));
+            }
+        } else {
+            recs.add(recOf(exchange.getMessage()));
+        }
+        inRoute++;
+        try {
+            processRecords(exchange, recs, manual);
+        } finally {
+            inRoute--;
+        }
+    }
+
+    private void processRecords(Exchange exchange, List<FakeKafka.Rec> recs, 
boolean manual) {
+        String ids = recs.stream().map(r -> r.id).toList().toString();
+        for (FakeKafka.Rec r : recs) {
+            r.routed++;
+            ledger.record("routed", "p" + r.partition, r.id);
+        }
+        if (faults && dst.chance(0.1)) {
+            dst.block(dst.between(1000, 10_000), "slow processing of " + ids);
+        } else {
+            dst.block(dst.between(0, 100), "processing of " + ids);
+        }
+        if (faults && dst.chance(processFailRate)) {
+            for (FakeKafka.Rec r : recs) {
+                r.failed++;
+                ledger.record("failed", "p" + r.partition, r.id);
+            }
+            throw Dst.injected(new IllegalStateException("dst: processing 
failed for " + ids));
+        }
+        for (FakeKafka.Rec r : recs) {
+            r.completed++;
+            ledger.record("completed", "p" + r.partition, r.id);
+        }
+        if (manual) {
+            Message m = exchange.getMessage();
+            boolean last = 
Boolean.TRUE.equals(m.getHeader(KafkaConstants.LAST_RECORD_BEFORE_COMMIT, 
Boolean.class));
+            if (commitEveryRecord || last) {
+                // a commit that fails fails the exchange, after the record 
was processed
+                m.getHeader(KafkaConstants.MANUAL_COMMIT, 
KafkaManualCommit.class).commit();
+            }
+        }
+    }
+
+    private FakeKafka.Rec recOf(Message message) {
+        int p = message.getHeader(KafkaConstants.PARTITION, Integer.class);
+        long offset = message.getHeader(KafkaConstants.OFFSET, Long.class);
+        FakeKafka.Rec r = kafka.rec(p, offset);
+        if (!r.id.equals(message.getBody(String.class))) {
+            violations.add("record p" + p + "@" + offset + " has the body " + 
message.getBody() + ", not " + r.id);
+        }
+        return r;
+    }
+
+    private String notDone() {
+        List<String> missing = new ArrayList<>();
+        for (List<FakeKafka.Rec> partitionLog : kafka.log) {
+            for (FakeKafka.Rec r : partitionLog) {
+                if (!kafka.done(r)) {
+                    missing.add(r.id + "@p" + r.partition + "/" + r.offset + " 
routed=" + r.routed + " failed=" + r.failed);
+                }
+            }
+        }
+        return missing.toString();
+    }
+
+    private boolean allDone() {
+        for (List<FakeKafka.Rec> partitionLog : kafka.log) {
+            for (FakeKafka.Rec r : partitionLog) {
+                if (!kafka.done(r)) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Waits for the sends and operations, switches the faults off, publishes 
one more record to each partition (its
+     * commit covers the records before it), waits for everything to be 
finished and committed (or a deadline), takes
+     * the final state and stops the route.
+     */
+    private void finisher(DefaultCamelContext context, DstExecutor operator, 
long deadline) {
+        if (deadline == -1 && (sendsLeft > 0 || opsLeft > 0)) {
+            operator.schedule("finisher", 500, () -> finisher(context, 
operator, -1));
+            return;
+        }
+        if (deadline == -1) {
+            long slow = dst.between(0, 10_000);
+            dst.trace("world: operations done, faults for another " + slow + " 
ms");
+            operator.schedule("finisher", slow, () -> finisher(context, 
operator, -2));
+            return;
+        }
+        if (deadline == -2) {
+            faults = false;
+            kafka.faults = false;
+            kafka.phantomLeaves();
+            for (int p = 0; p < partitions; p++) {
+                kafka.publish(p, "end" + p);
+            }
+            dst.trace("world: faults cleared");
+            long d = dst.now() + 10 * 60_000L;
+            operator.schedule("finisher", 500, () -> finisher(context, 
operator, d));
+            return;
+        }
+        ServiceStatus status = 
context.getRouteController().getRouteStatus(ROUTE);
+        boolean done = allDone();
+        boolean committed = kafka.allCommitted();
+        if ((inRoute > 0 || !done || !committed || status != 
ServiceStatus.Started || kafka.open.size() != 1)
+                && dst.now() < deadline) {
+            operator.schedule("finisher", 500, () -> finisher(context, 
operator, deadline));
+            return;
+        }
+        finalStatus = String.valueOf(status);
+        finalDone = done;
+        finalCommitted = committed;
+        finalOpenCount = kafka.open.size();
+        finalOpen = kafka.openNames();
+        if (inRoute > 0) {
+            // a stop would wait for it for ever (one-stack model)
+            finalDone = false;
+            finished = true;
+            return;
+        }
+        try {
+            context.getRouteController().stopRoute(ROUTE);
+        } catch (Exception e) {
+            violations.add("final stop failed: " + e);
+        }
+        finished = true;
+    }
+
+    /** The operator starts the route; a failed start is retried, as the 
supervising route controller does. */
+    private void startRoute(DefaultCamelContext context, DstExecutor operator) 
{
+        try {
+            context.getRouteController().startRoute(ROUTE);
+            ledger.record("routeStarted", "r", ROUTE);
+        } catch (Exception e) {
+            ledger.record("routeStartFailed", "r", ROUTE);
+            dst.trace("world: start failed (" + e.getMessage() + "), retry in 
2 s");
+            operator.schedule("retry start route", 2000, () -> 
startRoute(context, operator));
+        }
+    }
+
+    /**
+     * Starts the route again once the Kafka consumer of the stopped route is 
closed: its thread ends only after the
+     * stop returned (one-stack model), and a new consumer thread started 
above it would never let it resume.
+     */
+    private void startWhenClosed(DefaultCamelContext context, DstExecutor 
operator, long since) {
+        if (!kafka.open.isEmpty() && dst.now() - since < 10 * 60_000L) {
+            operator.schedule("start route when the consumer closed", 200,
+                    () -> startWhenClosed(context, operator, since));
+            return;
+        }
+        if (!kafka.open.isEmpty()) {
+            violations.add("Kafka consumers " + kafka.openNames() + " still 
open 10 minutes after the route stopped");
+        }
+        startRoute(context, operator);
+        opsLeft--;
+    }
+
+    private void planSends() {
+        DstExecutor producer = dst.executor("producer", 1);
+        long t = dst.between(0, 500);
+        int n = 0;
+        int bursts = (int) dst.between(1, 4);
+        for (int b = 0; b < bursts; b++) {
+            int size = (int) dst.between(1, 15);
+            for (int i = 0; i < size; i++) {
+                String id = "m" + n++;
+                int p = partitions == 1 ? 0 : dst.nextInt(partitions);
+                sendsLeft++;
+                producer.schedule("publish " + id, t, () -> {
+                    kafka.publish(p, id);
+                    sendsLeft--;
+                });
+                t += dst.between(0, 100);
+            }
+            t += dst.between(0, 8000);
+        }
+    }
+
+    private void planOperations(DefaultCamelContext context, DstExecutor 
operator) {
+        int ops = (int) dst.between(0, 4);
+        long t = dst.between(500, 3000);
+        for (int i = 0; i < ops; i++) {
+            int kind = dst.nextInt(3);
+            opsLeft++;
+            if (kind == 0) {
+                operator.schedule("stop route", t, () -> 
whenConnected(operator, "stop route", () -> {
+                    if (context.getRouteController().getRouteStatus(ROUTE) != 
ServiceStatus.Started) {
+                        opsLeft--;
+                        return;
+                    }
+                    try {
+                        context.getRouteController().stopRoute(ROUTE);
+                        ledger.record("routeStopped", "r", ROUTE);
+                    } catch (Exception e) {
+                        violations.add("stop failed: " + e);
+                    }
+                    long since = dst.now();
+                    operator.schedule("start route when the consumer closed", 
dst.between(0, 3000),
+                            () -> startWhenClosed(context, operator, since));
+                }));
+            } else if (kind == 1) {
+                // a second suspend and resume while the consumer may still 
handle the first resume (it reads the
+                // committed offsets before it resumes its partitions)
+                int times = dst.chance(0.5) ? 2 : 1;
+                operator.schedule("suspend route", t, () -> 
suspendAndResume(context, operator, times));
+            } else {
+                Set<Integer> taken = partitions == 1 ? Set.of(0) : 
dst.pick(List.of(Set.of(0), Set.of(1), Set.of(0, 1)));
+                long held = dst.between(500, 15_000);
+                operator.schedule("a member joins the group", t, () -> {
+                    kafka.phantomJoins(taken);
+                    operator.schedule("the member leaves the group", held, () 
-> {
+                        kafka.phantomLeaves();
+                        opsLeft--;
+                    });
+                });
+            }
+            t += dst.between(500, 10_000);
+        }
+    }
+
+    /**
+     * Runs an operation of the operator once no exchange is in the route. The 
operator task can run nested above an
+     * exchange that blocks (on the one stack), so it waits in a new task, not 
in this one.
+     */
+    private void whenIdle(DstExecutor operator, String what, Runnable 
operation) {
+        if (inRoute > 0) {
+            operator.schedule(what + " (an exchange is in the route)", 50, () 
-> whenIdle(operator, what, operation));
+            return;
+        }
+        operation.run();
+    }
+
+    /**
+     * Stops the route only while the consumer thread holds a Kafka consumer, 
not while it waits for the background task
+     * that creates one: a stop interrupts that wait with real threads 
(shutdownNow), which the one-stack model does not
+     * do, and the consumer thread would go on after the route was started 
again.
+     */
+    private void whenConnected(DstExecutor operator, String what, Runnable 
operation) {
+        if (kafka.open.size() != 1 || inRoute > 0) {
+            operator.schedule(what + " (no Kafka consumer, or an exchange is 
in the route)", 100,
+                    () -> whenConnected(operator, what, operation));
+            return;
+        }
+        operation.run();
+    }
+
+    private void suspendAndResume(DefaultCamelContext context, DstExecutor 
operator, int times) {
+        whenIdle(operator, "suspend route", () -> {
+            if (context.getRouteController().getRouteStatus(ROUTE) != 
ServiceStatus.Started) {
+                opsLeft--;
+                return;
+            }
+            try {
+                suspend(context);
+            } catch (Exception e) {
+                violations.add("suspend failed: " + e);
+                opsLeft--;
+                return;
+            }
+            operator.schedule("resume route", dst.between(100, 5000), () -> {
+                try {
+                    resume(context);
+                } catch (Exception e) {
+                    violations.add("resume failed: " + e);
+                }
+                if (times > 1) {
+                    operator.schedule("suspend route again", dst.between(0, 
pollTimeoutMs + 2000),
+                            () -> suspendAndResume(context, operator, times - 
1));
+                } else {
+                    opsLeft--;
+                }
+            });
+        });
+    }
+
+    private void suspend(DefaultCamelContext context) throws Exception {
+        context.getRouteController().suspendRoute(ROUTE);
+        kafka.suspendedSince = dst.now();
+        ledger.record("routeSuspended", "r", ROUTE);
+    }
+
+    private void resume(DefaultCamelContext context) throws Exception {
+        kafka.suspendedSince = -1;
+        context.getRouteController().resumeRoute(ROUTE);

Review Comment:
   ⚠️ **Exception safety in `resume()`**: `suspendedSince` is cleared on line 
553 **before** `resumeRoute()` is called on line 554. If `resumeRoute()` 
throws, `suspendedSince` will be `-1` even though the route never actually 
resumed — the `checkSuspended()` invariant check will then report the route as 
not suspended when it still is.
   
   Consider swapping the two lines:
   
   ```suggestion
           context.getRouteController().resumeRoute(ROUTE);
           kafka.suspendedSince = -1;
           ledger.record("routeResumed", "r", ROUTE);
   ```



##########
components/camel-kafka/src/test/java/org/apache/camel/component/kafka/dst/FakeKafka.java:
##########
@@ -0,0 +1,824 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.kafka.dst;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.camel.test.dst.Dst;
+import org.apache.camel.test.dst.DstLedger;
+import org.apache.kafka.clients.consumer.CommitFailedException;
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.clients.consumer.OffsetCommitCallback;
+import org.apache.kafka.clients.consumer.RetriableCommitFailedException;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.InterruptException;
+import org.apache.kafka.common.errors.TimeoutException;
+import org.apache.kafka.common.errors.WakeupException;
+import org.apache.kafka.common.header.internals.RecordHeaders;
+import org.apache.kafka.common.record.TimestampType;
+
+/**
+ * A fake Kafka cluster for one topic with one or two partitions and one 
consumer group, and fake consumers behind the
+ * {@link Consumer} interface (dynamic proxies), as camel-kafka gets them from 
its {@code KafkaClientFactory}.
+ * <p>
+ * Written from the Javadoc of {@code 
org.apache.kafka.clients.consumer.KafkaConsumer} and {@code Consumer}
+ * (kafka-clients 4.3) for the classic group protocol with an eager assignor 
(the default {@code RangeAssignor}).
+ * kafka-clients' {@code MockConsumer} is not used: its poll returns at once, 
its commits never fail or take time, and
+ * it has no group (no rebalance of its own), so it could not drive the timing 
this world is about. When kafka-clients
+ * is upgraded, check the rules below against the new Javadoc.
+ * <p>
+ * Rules (quotes from the KafkaConsumer Javadoc):
+ * <ul>
+ * <li>[P] poll: "fetch data for the topics or partitions specified using one 
of the subscribe/assign APIs"; it returns
+ * when records are available or the timeout expired, at most max.poll.records 
records ("fewer records might be
+ * returned"), from the position of each assigned partition that is not 
paused; the position then moves past them. "It
+ * is an error to not have subscribed to any topics or partitions before 
polling for data" (IllegalStateException).</li>
+ * <li>[G] group management: "the consumer will ... rebalance partitions among 
the members of the group" when a member
+ * joins or leaves; the rebalance happens inside poll, where the listener is 
called: "onPartitionsRevoked ... will be
+ * called ... during a rebalance operation when the consumer has to give up 
some partitions" (eager: every partition is
+ * revoked, then the new assignment is given to onPartitionsAssigned). 
"Rebalance will not preserve the pause/resume
+ * state" (pause). After an assignment the position of a partition is its 
committed offset, else auto.offset.reset
+ * (earliest in this world).</li>
+ * <li>[A] enable.auto.commit: "the committed offset will be the position", 
committed "with a frequency controlled by
+ * the config auto.commit.interval.ms" while the consumer polls, before a 
rebalance revokes the partitions, and when it
+ * closes. ("you must consume all data returned from each call to 
poll(Duration) before any subsequent calls, or before
+ * closing the consumer. If you fail to do either of these, it is possible for 
the committed offset to get ahead of the
+ * consumed position, which results in missing records.")</li>
+ * <li>[S] commitSync: "This is a synchronous commit and will block until 
either the commit succeeds, an unrecoverable
+ * error is encountered ... or the timeout ... expires". TimeoutException "if 
the timeout specified by
+ * default.api.timeout.ms expires before successful completion of the offset 
commit": whether the broker applied it is
+ * not known, so the seed chooses (assumed: the Javadoc does not say). A 
commit that takes longer than its timeout ends
+ * the same way. CommitFailedException "when you are trying to commit to 
partitions that are no longer assigned to this
+ * consumer". Until the member rejoins the group in its next poll, the group 
is preparing the rebalance and takes the
+ * commits of the partitions the member still owns (assumed from the classic 
protocol: the coordinator accepts commits
+ * of the current generation then). "This is a synchronous commit and will 
also invoke any pending offset commit
+ * callbacks of async commits".</li>
+ * <li>[C] commitAsync: "Offsets committed through multiple calls to this API 
are guaranteed to be sent in the same
+ * order as the invocations. Corresponding commit callbacks are also invoked 
in the same order"; the callback is invoked
+ * from a later poll, commitSync, commitAsync or close of the same consumer; a 
failed commit is not applied
+ * (RetriableCommitFailedException or CommitFailedException in the 
callback).</li>
+ * <li>[R] committed: "This call will block to do a remote call to get the 
latest committed offsets from the server";
+ * null for a partition without a committed offset.</li>
+ * <li>[W] wakeup: "The thread which is blocking in an operation will throw 
WakeupException. If no thread is blocking in
+ * a method which can throw WakeupException, the next call to such method will 
raise it instead." Here: poll, commitSync
+ * and committed.</li>
+ * <li>[X] seek, pause and resume of a partition that is not assigned: 
IllegalStateException ("No current assignment for
+ * partition"). unsubscribe leaves the group (revocation callback first); 
close commits with auto-commit, completes the
+ * pending async commits, and leaves the group.</li>
+ * <li>[L] an exception of the rebalance listener: the client gives up the 
partitions (or takes the new ones) first,
+ * then poll, unsubscribe or close throws it, wrapped in a KafkaException 
("User rebalance callback throws an error"); a
+ * WakeupException or InterruptException is thrown at once, and the member 
keeps its partitions (assumed from the
+ * classic client; the Javadoc says only that the exception is propagated). 
close turns wakeups off first, as the
+ * classic client does.</li>
+ * </ul>
+ * A phantom member can join the group (and leave again): it takes partitions 
in the rebalance, but does not consume or
+ * commit, so its partitions wait for the Camel consumer to get them back.
+ * <p>
+ * Timing (one-stack rule): poll waits first and fetches at the end of its 
wait; the rebalance in poll blocks for the
+ * join before the listener is called; commits take effect at the start of 
their latency; committed blocks, then reads.
+ * The other calls do not block.
+ * <p>
+ * Not modelled: several topics, topic patterns and assign(), 
max.poll.interval.ms and session timeouts (a member is
+ * never removed for being slow), the cooperative rebalance protocol and 
group.protocol=consumer, transactions and
+ * isolation levels, leader epochs, log retention and offsets out of range.
+ * <p>
+ * The fake checks the at-least-once contract of the consumer when the group 
takes a commit: a committed offset must not
+ * pass a record that the route has not finished with ({@link 
#done(TopicPartition, long)}).
+ */
+final class FakeKafka {
+
+    static final String TOPIC = "dst";
+    static final String GROUP = "g";
+
+    /** A record of the topic, and what the route did with it. */
+    static final class Rec {
+        final int partition;
+        final long offset;
+        final String id;
+        int routed;
+        int completed;
+        int failed;
+
+        Rec(int partition, long offset, String id) {
+            this.partition = partition;
+            this.offset = offset;
+            this.id = id;
+        }
+    }
+
+    /** An async commit waiting for its callback. */
+    private record PendingCallback(Map<TopicPartition, OffsetAndMetadata> 
offsets, Exception error,
+            OffsetCommitCallback callback) {
+    }
+
+    final class Member implements InvocationHandler {
+        final String name;
+        final Set<TopicPartition> owned = new TreeSet<>(FakeKafka::compare);
+        final Set<TopicPartition> paused = new TreeSet<>(FakeKafka::compare);
+        final Map<TopicPartition, Long> position = new 
TreeMap<>(FakeKafka::compare);
+        final List<PendingCallback> callbacks = new ArrayList<>();
+        ConsumerRebalanceListener listener;
+        boolean subscribed;
+        boolean closed;
+        boolean wakeup;
+        long generation = -1;
+        long lastAutoCommit;
+        // for the check of a suspended route: when the last poll started, and 
whether it returned normally
+        long lastPollStart = -1;
+        boolean lastPollReturned;
+
+        Member(String name) {
+            this.name = name;
+        }
+
+        @Override
+        public Object invoke(Object proxy, Method method, Object[] args) {
+            String m = method.getName();
+            switch (m) {
+                case "toString":
+                    return "FakeKafkaConsumer[" + name + "]";
+                case "hashCode":
+                    return System.identityHashCode(this);
+                case "equals":
+                    return proxy == args[0];
+                case "wakeup":
+                    dst.trace("fake: " + name + " wakeup");
+                    wakeup = true;
+                    return null;
+                case "close":
+                    close();
+                    return null;
+                case "metrics":
+                    return Map.of();
+                default:
+                    break;
+            }
+            checkOpen(m);
+            switch (m) {
+                case "subscribe":
+                    subscribe(args);
+                    return null;
+                case "unsubscribe":
+                    unsubscribe();
+                    return null;
+                case "subscription":
+                    return subscribed ? Set.of(TOPIC) : Set.of();
+                case "assignment":
+                    return copyOf(owned);
+                case "paused":
+                    return copyOf(paused);
+                case "poll":
+                    return poll((Duration) args[0]);
+                case "commitSync":
+                    commitSync(args);
+                    return null;
+                case "commitAsync":
+                    commitAsync(args);
+                    return null;
+                case "committed":
+                    return committed(args);
+                case "seek":
+                    seek((TopicPartition) args[0],
+                            args[1] instanceof OffsetAndMetadata o ? 
o.offset() : (Long) args[1]);
+                    return null;
+                case "pause":
+                    pause(args, true);
+                    return null;
+                case "resume":
+                    pause(args, false);
+                    return null;
+                default:
+                    violations.add(name + " called " + m + " (not modelled)");
+                    throw new UnsupportedOperationException(m);
+            }
+        }
+
+        private void checkOpen(String m) {
+            if (closed) {
+                violations.add(name + " used after close: " + m);
+                throw new IllegalStateException("This consumer has already 
been closed.");
+            }
+        }
+
+        private void checkWakeup() {
+            if (wakeup) {
+                wakeup = false;
+                dst.trace("fake: " + name + " throws WakeupException");
+                throw new WakeupException();
+            }
+        }
+
+        private void subscribe(Object[] args) {
+            if (!(args[0] instanceof Collection<?> topics)) {
+                violations.add(name + " subscribed with " + 
args[0].getClass().getSimpleName() + " (not modelled)");
+                throw new UnsupportedOperationException("subscribe(" + args[0] 
+ ")");
+            }
+            if (!topics.equals(List.of(TOPIC)) && 
!topics.equals(Set.of(TOPIC))) {
+                violations.add(name + " subscribed to " + topics);
+            }
+            listener = args.length > 1 ? (ConsumerRebalanceListener) args[1] : 
null;
+            if (!subscribed) {
+                subscribed = true;
+                members.add(this);
+                groupChanged(name + " joins");
+            }
+        }
+
+        private void unsubscribe() {
+            if (!subscribed) {
+                return;
+            }
+            dst.trace("fake: " + name + " unsubscribe");
+            leave();
+        }
+
+        /** [X] leaves the group: the revocation callback first, then the 
partitions are given up; [L]. */
+        private void leave() {
+            RuntimeException failure = null;
+            if (!owned.isEmpty() && listener != null) {
+                failure = revoke();
+                if (interruption(failure)) {
+                    // [L] thrown at once: the member keeps its partitions and 
stays in the group
+                    throw failure;
+                }
+            }
+            owned.clear();
+            paused.clear();
+            position.clear();
+            subscribed = false;
+            members.remove(this);
+            groupChanged(name + " leaves");
+            if (failure != null) {
+                throw listenerFailure(failure);
+            }
+        }
+
+        /** The revocation callback; its exception is returned, to be thrown 
once the partitions are given up [L]. */
+        private RuntimeException revoke() {
+            try {
+                listener.onPartitionsRevoked(copyOf(owned));
+                return null;
+            } catch (RuntimeException e) {
+                dst.trace("fake: " + name + " the revocation callback throws " 
+ e);
+                return e;
+            }
+        }
+
+        private RuntimeException listenerFailure(RuntimeException e) {
+            if (interruption(e)) {
+                return e;
+            }
+            return new KafkaException("User rebalance callback throws an 
error", e);
+        }
+
+        private boolean interruption(RuntimeException e) {
+            return e instanceof WakeupException || e instanceof 
InterruptException;
+        }
+
+        private ConsumerRecords<Object, Object> poll(Duration timeout) {
+            invokeCallbacks();
+            checkWakeup();
+            if (!subscribed) {
+                violations.add(name + " polled without a subscription");
+                throw new IllegalStateException("Consumer is not subscribed to 
any topics or assigned any partitions");
+            }
+            checkSuspended();
+            lastPollStart = dst.now();
+            lastPollReturned = false;
+            rebalanceIfNeeded();
+            // [A] the auto-commit of the positions, after the group was 
joined and before the fetch
+            if (autoCommit && dst.now() - lastAutoCommit >= 
autoCommitIntervalMs) {
+                autoCommit("interval");
+            }
+            if (faults && dst.chance(pollFailRate)) {
+                dst.block(dst.between(1, 50), name + " poll fails");
+                ledger.record("pollFault", "c", name);
+                throw Dst.injected(new KafkaException("dst: poll failed"));
+            }
+            long waitMillis = Math.max(1, timeout.toMillis());
+            boolean ready = dst.awaitUntil(
+                    () -> wakeup || closed || fetchable() || generation != 
FakeKafka.this.generation, waitMillis,
+                    name + " poll");
+            if (ready && !wakeup && !closed) {
+                // reads take effect at the end of their latency
+                dst.block(dst.between(1, 20), name + " fetch");
+            }
+            if (closed) {
+                violations.add(name + " closed while it polls");
+                throw new IllegalStateException("This consumer has already 
been closed.");
+            }
+            checkWakeup();
+            rebalanceIfNeeded();
+            Map<TopicPartition, List<ConsumerRecord<Object, Object>>> map = 
new LinkedHashMap<>();
+            int budget = maxPollRecords;
+            if (budget > 1 && dst.chance(0.3)) {
+                // [P] "fewer records might be returned"
+                budget = (int) dst.between(1, budget - 1);
+            }
+            for (TopicPartition tp : owned) {
+                if (paused.contains(tp) || budget == 0) {
+                    continue;
+                }
+                List<Rec> partitionLog = log.get(tp.partition());
+                long pos = position.get(tp);
+                List<ConsumerRecord<Object, Object>> list = new ArrayList<>();
+                while (budget > 0 && pos < partitionLog.size()) {
+                    Rec r = partitionLog.get((int) pos);
+                    list.add(new ConsumerRecord<>(
+                            TOPIC, tp.partition(), r.offset, Dst.EPOCH_MILLIS 
+ dst.now(), TimestampType.CREATE_TIME, -1,
+                            -1, null, r.id, new RecordHeaders(), 
Optional.empty(), Optional.empty()));
+                    ledger.record("fetched", "p" + tp.partition(), r.id);
+                    pos++;
+                    budget--;
+                }
+                position.put(tp, pos);
+                if (!list.isEmpty()) {
+                    map.put(tp, list);
+                }
+            }
+            if (!map.isEmpty()) {
+                dst.trace("fake: " + name + " poll returns " + 
map.values().stream().mapToInt(List::size).sum()
+                          + " records " + positions());
+            }
+            lastPollReturned = true;
+            return new ConsumerRecords<>(map, Map.of());
+        }
+
+        /**
+         * The check of a suspended route: once the consumer completed a poll 
that started after the route was suspended
+         * (and then handled its records and its pause request), it must have 
paused its partitions.
+         */
+        private void checkSuspended() {
+            if (suspendedSince >= 0 && lastPollStart >= suspendedSince && 
lastPollReturned) {
+                Set<TopicPartition> fetching = new 
TreeSet<>(FakeKafka::compare);
+                fetching.addAll(owned);
+                fetching.removeAll(paused);
+                if (!fetching.isEmpty()) {
+                    suspendViolations.add(name + " still fetches " + fetching 
+ " at t=" + dst.now()
+                                          + " after the route was suspended at 
t=" + suspendedSince);
+                }
+            }
+        }
+
+        private String positions() {
+            return "positions=" + position;
+        }
+
+        private boolean fetchable() {
+            for (TopicPartition tp : owned) {
+                if (!paused.contains(tp) && position.get(tp) < 
log.get(tp.partition()).size()) {
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        /** [G] an eager rebalance inside poll, when the group changed since 
this member last joined. */
+        private void rebalanceIfNeeded() {
+            if (generation == FakeKafka.this.generation) {
+                return;
+            }
+            // the join and sync round trips of the group
+            dst.block(dst.between(1, 50), name + " rebalance");
+            if (closed || !subscribed) {
+                return;
+            }
+            if (autoCommit) {
+                autoCommit("before revocation");
+            }
+            if (!owned.isEmpty() && listener != null) {
+                RuntimeException failure = revoke();
+                if (interruption(failure)) {
+                    // [L] thrown at once: the member keeps its partitions and 
joins again in its next poll
+                    throw failure;
+                }
+                if (failure != null) {
+                    // [L] the partitions are given up, the member joins again 
in its next poll
+                    owned.clear();
+                    paused.clear();
+                    position.clear();
+                    throw listenerFailure(failure);
+                }
+            }
+            if (closed || !subscribed) {
+                return;
+            }
+            long gen = FakeKafka.this.generation;
+            Set<TopicPartition> assigned = assignmentOf(this);
+            owned.clear();
+            paused.clear();
+            position.clear();
+            owned.addAll(assigned);
+            for (TopicPartition tp : owned) {
+                Long c = committed.get(tp.partition());
+                position.put(tp, c != null ? c : 0L);
+            }
+            generation = gen;
+            dst.trace("fake: " + name + " assigned " + owned + " (generation " 
+ gen + ") " + positions());
+            if (listener != null) {
+                try {
+                    listener.onPartitionsAssigned(copyOf(owned));
+                } catch (RuntimeException e) {
+                    // [L] the partitions are taken, then poll throws
+                    throw listenerFailure(e);
+                }
+            }
+        }
+
+        /** [A] commits the positions (before the fetch of a poll, before a 
revocation, at close). */
+        private void autoCommit(String why) {
+            lastAutoCommit = dst.now();
+            if (owned.isEmpty()) {
+                return;
+            }
+            if (faults && dst.chance(commitFailRate)) {
+                dst.trace("fake: " + name + " auto-commit (" + why + ") 
fails");
+                ledger.record("commitFault", "c", name);
+                return;
+            }
+            Map<TopicPartition, OffsetAndMetadata> offsets = new 
TreeMap<>(FakeKafka::compare);
+            for (TopicPartition tp : owned) {
+                offsets.put(tp, new OffsetAndMetadata(position.get(tp)));
+            }
+            apply(this, offsets, "auto-commit (" + why + ")");
+        }
+
+        @SuppressWarnings("unchecked")
+        private void commitSync(Object[] args) {
+            invokeCallbacks();
+            checkWakeup();
+            Map<TopicPartition, OffsetAndMetadata> offsets = args != null && 
args.length > 0 && args[0] instanceof Map
+                    ? (Map<TopicPartition, OffsetAndMetadata>) args[0] : 
consumedPositions();
+            long timeout = defaultApiTimeoutMs;
+            if (args != null && args.length > 0 && args[args.length - 1] 
instanceof Duration d) {
+                timeout = d.toMillis();
+            }
+            checkCommittable(offsets);
+            long latency = commitLatency();
+            if (faults && dst.chance(commitFailRate) || latency > timeout) {

Review Comment:
   💡 **Parentheses for clarity**: `&&` has higher precedence than `||`, so this 
reads as `(faults && dst.chance(commitFailRate)) || (latency > timeout)`, which 
is likely the intended logic. Adding explicit parentheses would make that 
intent unambiguous and avoid any future misread.
   
   ```suggestion
               if ((faults && dst.chance(commitFailRate)) || latency > timeout) 
{
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to