This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new b90afd130f95 CAMEL-26947: camel-kafka - a DST fake world for the Kafka
consumer (#29992)
b90afd130f95 is described below
commit b90afd130f952badbc48ad6b082222858c9f0e23
Author: allthingssecurity <[email protected]>
AuthorDate: Mon Nov 16 20:33:01 2026 +0530
CAMEL-26947: camel-kafka - a DST fake world for the Kafka consumer (#29992)
[CAMEL-26947](https://issues.apache.org/jira/browse/CAMEL-26947), part of
CAMEL-26610.
A deterministic simulation test (camel-test-dst) of the plain camel-kafka
consumer, in the `dst` package of its tests, in the same form as the SQS and
camel-kafka-share worlds: `KafkaConsumerDstTest` (1000 seeds), `KafkaWorld` and
`FakeKafka`. Test only; no main code changes.
What the world models: the real consumer (`KafkaConsumer`,
`KafkaFetchRecords`, the commit managers) and Camel's routing run on the
virtual-time loop; only Kafka is faked, as a `Consumer` dynamic proxy returned
by a `KafkaClientFactory` set on the component. The fake is written from the
`KafkaConsumer` and `Consumer` Javadoc of kafka-clients 4.3, for the classic
group protocol with an eager assignor, and quotes the rules it follows: poll
with `max.poll.records`, pause and resume, seek [...]
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../org/apache/camel/catalog/components/kafka.json | 3 +
.../org/apache/camel/catalog/docs/test-dst.adoc | 12 +
components/camel-kafka/pom.xml | 25 +
.../org/apache/camel/component/kafka/kafka.json | 3 +
.../camel/component/kafka/dst/FakeKafka.java | 824 +++++++++++++++++++++
.../component/kafka/dst/KafkaConsumerDstTest.java | 41 +
.../camel/component/kafka/dst/KafkaWorld.java | 557 ++++++++++++++
.../src/test/resources/log4j2.properties | 9 +-
.../camel-test-dst/src/main/docs/test-dst.adoc | 12 +
9 files changed, 1485 insertions(+), 1 deletion(-)
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/kafka.json
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/kafka.json
index df504340987d..d051d1c23d53 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/kafka.json
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/kafka.json
@@ -299,5 +299,8 @@
"sslTruststoreLocation": { "index": 119, "kind": "parameter",
"displayName": "Ssl Truststore Location", "group": "security", "label":
"common,security", "required": false, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "autowired": false, "secret": false,
"configurationClass": "org.apache.camel.component.kafka.KafkaConfiguration",
"configurationField": "configuration", "description": "The location of the
trust store file." },
"sslTruststorePassword": { "index": 120, "kind": "parameter",
"displayName": "Ssl Truststore Password", "group": "security", "label":
"common,security", "required": false, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "autowired": false, "secret": true,
"security": "secret", "configurationClass":
"org.apache.camel.component.kafka.KafkaConfiguration", "configurationField":
"configuration", "description": "The password for the trust store file. If a
password is [...]
"sslTruststoreType": { "index": 121, "kind": "parameter", "displayName":
"Ssl Truststore Type", "group": "security", "label": "common,security",
"required": false, "type": "string", "javaType": "java.lang.String",
"deprecated": false, "autowired": false, "secret": false, "defaultValue":
"JKS", "configurationClass":
"org.apache.camel.component.kafka.KafkaConfiguration", "configurationField":
"configuration", "description": "The file format of the trust store file. The
default value is [...]
+ },
+ "dstWorlds": {
+ "KafkaWorld": { "javaType":
"org.apache.camel.component.kafka.dst.KafkaWorld", "fake": "FakeKafka: a topic
with one or two partitions and a consumer group (classic protocol, eager
rebalance), behind the Consumer interface of KafkaClientFactory", "covers": [
"commits by Kafka's auto-commit, by manual synchronous or asynchronous commits
in the route, and per batch with batching=true", "breakOnFirstError",
"pollOnError ERROR_HANDLER, DISCARD, RETRY and RECONNECT", "a poll that fails
whi [...]
}
}
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/test-dst.adoc
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/test-dst.adoc
index 0fe66cfbe03a..4923ed2b2935 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/test-dst.adoc
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/test-dst.adoc
@@ -101,6 +101,18 @@ a|* synchronousPull with ackMode AUTO, and ackMode NONE
with grouped messages ac
* a stop and start of the same consumer in one step, as a throttling route
policy does
|CAMEL-25712, CAMEL-25790, CAMEL-26647
+|camel-kafka
+|`KafkaWorld` (`KafkaConsumerDstTest`)
+|FakeKafka: a topic with one or two partitions and a consumer group (classic
protocol, eager rebalance), behind the Consumer interface of KafkaClientFactory
+a|* commits by Kafka's auto-commit, by manual synchronous or asynchronous
commits in the route, and per batch with batching=true
+* breakOnFirstError
+* pollOnError ERROR_HANDLER, DISCARD, RETRY and RECONNECT
+* a poll that fails while a batch is incomplete
+* route restart, also while the consumer polls
+* route suspend and resume, with a second request while the consumer handles
the first
+* partitions revoked and assigned again by a rebalance while records are
processed
+|CAMEL-25313, CAMEL-25756
+
|camel-kafka-share
|`KafkaShareWorld` (`KafkaShareConsumerDstTest`)
|FakeShareGroup: a share group (KIP-932) of one topic with one partition,
behind ShareConsumer
diff --git a/components/camel-kafka/pom.xml b/components/camel-kafka/pom.xml
index d8c8e455a2eb..9d5e76b9719c 100644
--- a/components/camel-kafka/pom.xml
+++ b/components/camel-kafka/pom.xml
@@ -37,6 +37,8 @@
<!-- the ITs use less than 600 MB per fork: do not reserve the 3G of
the parent. Redefined entirely (and not
through additional-vmargs) to avoid having two -Xmx on the
command line -->
<camel.failsafe.fork.vmargs>-XX:+ExitOnOutOfMemoryError -Xmx1G
${camel.failsafe.fork.additional-vmargs}</camel.failsafe.fork.vmargs>
+ <!-- the DST tests load the byte code agent of camel-test-dst when the
JVM starts (see test-dst.adoc) -->
+
<camel.surefire.fork.additional-vmargs>"-javaagent:${net.bytebuddy:byte-buddy-agent:jar}"</camel.surefire.fork.additional-vmargs>
</properties>
<dependencies>
@@ -175,8 +177,31 @@
<version>${mockito-version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-test-dst</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <!-- sets the property net.bytebuddy:byte-buddy-agent:jar to
the path of the agent of the DST tests -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>dst-agent-path</id>
+ <goals>
+ <goal>properties</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
<profiles>
<profile>
<!-- This is needed only for camel-kafka to skip itests selected
in the backwards-compatibility profile -->
diff --git
a/components/camel-kafka/src/generated/resources/META-INF/org/apache/camel/component/kafka/kafka.json
b/components/camel-kafka/src/generated/resources/META-INF/org/apache/camel/component/kafka/kafka.json
index df504340987d..d051d1c23d53 100644
---
a/components/camel-kafka/src/generated/resources/META-INF/org/apache/camel/component/kafka/kafka.json
+++
b/components/camel-kafka/src/generated/resources/META-INF/org/apache/camel/component/kafka/kafka.json
@@ -299,5 +299,8 @@
"sslTruststoreLocation": { "index": 119, "kind": "parameter",
"displayName": "Ssl Truststore Location", "group": "security", "label":
"common,security", "required": false, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "autowired": false, "secret": false,
"configurationClass": "org.apache.camel.component.kafka.KafkaConfiguration",
"configurationField": "configuration", "description": "The location of the
trust store file." },
"sslTruststorePassword": { "index": 120, "kind": "parameter",
"displayName": "Ssl Truststore Password", "group": "security", "label":
"common,security", "required": false, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "autowired": false, "secret": true,
"security": "secret", "configurationClass":
"org.apache.camel.component.kafka.KafkaConfiguration", "configurationField":
"configuration", "description": "The password for the trust store file. If a
password is [...]
"sslTruststoreType": { "index": 121, "kind": "parameter", "displayName":
"Ssl Truststore Type", "group": "security", "label": "common,security",
"required": false, "type": "string", "javaType": "java.lang.String",
"deprecated": false, "autowired": false, "secret": false, "defaultValue":
"JKS", "configurationClass":
"org.apache.camel.component.kafka.KafkaConfiguration", "configurationField":
"configuration", "description": "The file format of the trust store file. The
default value is [...]
+ },
+ "dstWorlds": {
+ "KafkaWorld": { "javaType":
"org.apache.camel.component.kafka.dst.KafkaWorld", "fake": "FakeKafka: a topic
with one or two partitions and a consumer group (classic protocol, eager
rebalance), behind the Consumer interface of KafkaClientFactory", "covers": [
"commits by Kafka's auto-commit, by manual synchronous or asynchronous commits
in the route, and per batch with batching=true", "breakOnFirstError",
"pollOnError ERROR_HANDLER, DISCARD, RETRY and RECONNECT", "a poll that fails
whi [...]
}
}
diff --git
a/components/camel-kafka/src/test/java/org/apache/camel/component/kafka/dst/FakeKafka.java
b/components/camel-kafka/src/test/java/org/apache/camel/component/kafka/dst/FakeKafka.java
new file mode 100644
index 000000000000..87d82cdcca98
--- /dev/null
+++
b/components/camel-kafka/src/test/java/org/apache/camel/component/kafka/dst/FakeKafka.java
@@ -0,0 +1,824 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.kafka.dst;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.camel.test.dst.Dst;
+import org.apache.camel.test.dst.DstLedger;
+import org.apache.kafka.clients.consumer.CommitFailedException;
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.clients.consumer.OffsetCommitCallback;
+import org.apache.kafka.clients.consumer.RetriableCommitFailedException;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.InterruptException;
+import org.apache.kafka.common.errors.TimeoutException;
+import org.apache.kafka.common.errors.WakeupException;
+import org.apache.kafka.common.header.internals.RecordHeaders;
+import org.apache.kafka.common.record.TimestampType;
+
+/**
+ * A fake Kafka cluster for one topic with one or two partitions and one
consumer group, and fake consumers behind the
+ * {@link Consumer} interface (dynamic proxies), as camel-kafka gets them from
its {@code KafkaClientFactory}.
+ * <p>
+ * Written from the Javadoc of {@code
org.apache.kafka.clients.consumer.KafkaConsumer} and {@code Consumer}
+ * (kafka-clients 4.3) for the classic group protocol with an eager assignor
(the default {@code RangeAssignor}).
+ * kafka-clients' {@code MockConsumer} is not used: its poll returns at once,
its commits never fail or take time, and
+ * it has no group (no rebalance of its own), so it could not drive the timing
this world is about. When kafka-clients
+ * is upgraded, check the rules below against the new Javadoc.
+ * <p>
+ * Rules (quotes from the KafkaConsumer Javadoc):
+ * <ul>
+ * <li>[P] poll: "fetch data for the topics or partitions specified using one
of the subscribe/assign APIs"; it returns
+ * when records are available or the timeout expired, at most max.poll.records
records ("fewer records might be
+ * returned"), from the position of each assigned partition that is not
paused; the position then moves past them. "It
+ * is an error to not have subscribed to any topics or partitions before
polling for data" (IllegalStateException).</li>
+ * <li>[G] group management: "the consumer will ... rebalance partitions among
the members of the group" when a member
+ * joins or leaves; the rebalance happens inside poll, where the listener is
called: "onPartitionsRevoked ... will be
+ * called ... during a rebalance operation when the consumer has to give up
some partitions" (eager: every partition is
+ * revoked, then the new assignment is given to onPartitionsAssigned).
"Rebalance will not preserve the pause/resume
+ * state" (pause). After an assignment the position of a partition is its
committed offset, else auto.offset.reset
+ * (earliest in this world).</li>
+ * <li>[A] enable.auto.commit: "the committed offset will be the position",
committed "with a frequency controlled by
+ * the config auto.commit.interval.ms" while the consumer polls, before a
rebalance revokes the partitions, and when it
+ * closes. ("you must consume all data returned from each call to
poll(Duration) before any subsequent calls, or before
+ * closing the consumer. If you fail to do either of these, it is possible for
the committed offset to get ahead of the
+ * consumed position, which results in missing records.")</li>
+ * <li>[S] commitSync: "This is a synchronous commit and will block until
either the commit succeeds, an unrecoverable
+ * error is encountered ... or the timeout ... expires". TimeoutException "if
the timeout specified by
+ * default.api.timeout.ms expires before successful completion of the offset
commit": whether the broker applied it is
+ * not known, so the seed chooses (assumed: the Javadoc does not say). A
commit that takes longer than its timeout ends
+ * the same way. CommitFailedException "when you are trying to commit to
partitions that are no longer assigned to this
+ * consumer". Until the member rejoins the group in its next poll, the group
is preparing the rebalance and takes the
+ * commits of the partitions the member still owns (assumed from the classic
protocol: the coordinator accepts commits
+ * of the current generation then). "This is a synchronous commit and will
also invoke any pending offset commit
+ * callbacks of async commits".</li>
+ * <li>[C] commitAsync: "Offsets committed through multiple calls to this API
are guaranteed to be sent in the same
+ * order as the invocations. Corresponding commit callbacks are also invoked
in the same order"; the callback is invoked
+ * from a later poll, commitSync, commitAsync or close of the same consumer; a
failed commit is not applied
+ * (RetriableCommitFailedException or CommitFailedException in the
callback).</li>
+ * <li>[R] committed: "This call will block to do a remote call to get the
latest committed offsets from the server";
+ * null for a partition without a committed offset.</li>
+ * <li>[W] wakeup: "The thread which is blocking in an operation will throw
WakeupException. If no thread is blocking in
+ * a method which can throw WakeupException, the next call to such method will
raise it instead." Here: poll, commitSync
+ * and committed.</li>
+ * <li>[X] seek, pause and resume of a partition that is not assigned:
IllegalStateException ("No current assignment for
+ * partition"). unsubscribe leaves the group (revocation callback first);
close commits with auto-commit, completes the
+ * pending async commits, and leaves the group.</li>
+ * <li>[L] an exception of the rebalance listener: the client gives up the
partitions (or takes the new ones) first,
+ * then poll, unsubscribe or close throws it, wrapped in a KafkaException
("User rebalance callback throws an error"); a
+ * WakeupException or InterruptException is thrown at once, and the member
keeps its partitions (assumed from the
+ * classic client; the Javadoc says only that the exception is propagated).
close turns wakeups off first, as the
+ * classic client does.</li>
+ * </ul>
+ * A phantom member can join the group (and leave again): it takes partitions
in the rebalance, but does not consume or
+ * commit, so its partitions wait for the Camel consumer to get them back.
+ * <p>
+ * Timing (one-stack rule): poll waits first and fetches at the end of its
wait; the rebalance in poll blocks for the
+ * join before the listener is called; commits take effect at the start of
their latency; committed blocks, then reads.
+ * The other calls do not block.
+ * <p>
+ * Not modelled: several topics, topic patterns and assign(),
max.poll.interval.ms and session timeouts (a member is
+ * never removed for being slow), the cooperative rebalance protocol and
group.protocol=consumer, transactions and
+ * isolation levels, leader epochs, log retention and offsets out of range.
+ * <p>
+ * The fake checks the at-least-once contract of the consumer when the group
takes a commit: a committed offset must not
+ * pass a record that the route has not finished with ({@link
#done(TopicPartition, long)}).
+ */
+final class FakeKafka {
+
+ static final String TOPIC = "dst";
+ static final String GROUP = "g";
+
+ /** A record of the topic, and what the route did with it. */
+ static final class Rec {
+ final int partition;
+ final long offset;
+ final String id;
+ int routed;
+ int completed;
+ int failed;
+
+ Rec(int partition, long offset, String id) {
+ this.partition = partition;
+ this.offset = offset;
+ this.id = id;
+ }
+ }
+
+ /** An async commit waiting for its callback. */
+ private record PendingCallback(Map<TopicPartition, OffsetAndMetadata>
offsets, Exception error,
+ OffsetCommitCallback callback) {
+ }
+
+ final class Member implements InvocationHandler {
+ final String name;
+ final Set<TopicPartition> owned = new TreeSet<>(FakeKafka::compare);
+ final Set<TopicPartition> paused = new TreeSet<>(FakeKafka::compare);
+ final Map<TopicPartition, Long> position = new
TreeMap<>(FakeKafka::compare);
+ final List<PendingCallback> callbacks = new ArrayList<>();
+ ConsumerRebalanceListener listener;
+ boolean subscribed;
+ boolean closed;
+ boolean wakeup;
+ long generation = -1;
+ long lastAutoCommit;
+ // for the check of a suspended route: when the last poll started, and
whether it returned normally
+ long lastPollStart = -1;
+ boolean lastPollReturned;
+
+ Member(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public Object invoke(Object proxy, Method method, Object[] args) {
+ String m = method.getName();
+ switch (m) {
+ case "toString":
+ return "FakeKafkaConsumer[" + name + "]";
+ case "hashCode":
+ return System.identityHashCode(this);
+ case "equals":
+ return proxy == args[0];
+ case "wakeup":
+ dst.trace("fake: " + name + " wakeup");
+ wakeup = true;
+ return null;
+ case "close":
+ close();
+ return null;
+ case "metrics":
+ return Map.of();
+ default:
+ break;
+ }
+ checkOpen(m);
+ switch (m) {
+ case "subscribe":
+ subscribe(args);
+ return null;
+ case "unsubscribe":
+ unsubscribe();
+ return null;
+ case "subscription":
+ return subscribed ? Set.of(TOPIC) : Set.of();
+ case "assignment":
+ return copyOf(owned);
+ case "paused":
+ return copyOf(paused);
+ case "poll":
+ return poll((Duration) args[0]);
+ case "commitSync":
+ commitSync(args);
+ return null;
+ case "commitAsync":
+ commitAsync(args);
+ return null;
+ case "committed":
+ return committed(args);
+ case "seek":
+ seek((TopicPartition) args[0],
+ args[1] instanceof OffsetAndMetadata o ?
o.offset() : (Long) args[1]);
+ return null;
+ case "pause":
+ pause(args, true);
+ return null;
+ case "resume":
+ pause(args, false);
+ return null;
+ default:
+ violations.add(name + " called " + m + " (not modelled)");
+ throw new UnsupportedOperationException(m);
+ }
+ }
+
+ private void checkOpen(String m) {
+ if (closed) {
+ violations.add(name + " used after close: " + m);
+ throw new IllegalStateException("This consumer has already
been closed.");
+ }
+ }
+
+ private void checkWakeup() {
+ if (wakeup) {
+ wakeup = false;
+ dst.trace("fake: " + name + " throws WakeupException");
+ throw new WakeupException();
+ }
+ }
+
+ private void subscribe(Object[] args) {
+ if (!(args[0] instanceof Collection<?> topics)) {
+ violations.add(name + " subscribed with " +
args[0].getClass().getSimpleName() + " (not modelled)");
+ throw new UnsupportedOperationException("subscribe(" + args[0]
+ ")");
+ }
+ if (!topics.equals(List.of(TOPIC)) &&
!topics.equals(Set.of(TOPIC))) {
+ violations.add(name + " subscribed to " + topics);
+ }
+ listener = args.length > 1 ? (ConsumerRebalanceListener) args[1] :
null;
+ if (!subscribed) {
+ subscribed = true;
+ members.add(this);
+ groupChanged(name + " joins");
+ }
+ }
+
+ private void unsubscribe() {
+ if (!subscribed) {
+ return;
+ }
+ dst.trace("fake: " + name + " unsubscribe");
+ leave();
+ }
+
+ /** [X] leaves the group: the revocation callback first, then the
partitions are given up; [L]. */
+ private void leave() {
+ RuntimeException failure = null;
+ if (!owned.isEmpty() && listener != null) {
+ failure = revoke();
+ if (interruption(failure)) {
+ // [L] thrown at once: the member keeps its partitions and
stays in the group
+ throw failure;
+ }
+ }
+ owned.clear();
+ paused.clear();
+ position.clear();
+ subscribed = false;
+ members.remove(this);
+ groupChanged(name + " leaves");
+ if (failure != null) {
+ throw listenerFailure(failure);
+ }
+ }
+
+ /** The revocation callback; its exception is returned, to be thrown
once the partitions are given up [L]. */
+ private RuntimeException revoke() {
+ try {
+ listener.onPartitionsRevoked(copyOf(owned));
+ return null;
+ } catch (RuntimeException e) {
+ dst.trace("fake: " + name + " the revocation callback throws "
+ e);
+ return e;
+ }
+ }
+
+ private RuntimeException listenerFailure(RuntimeException e) {
+ if (interruption(e)) {
+ return e;
+ }
+ return new KafkaException("User rebalance callback throws an
error", e);
+ }
+
+ private boolean interruption(RuntimeException e) {
+ return e instanceof WakeupException || e instanceof
InterruptException;
+ }
+
+ private ConsumerRecords<Object, Object> poll(Duration timeout) {
+ invokeCallbacks();
+ checkWakeup();
+ if (!subscribed) {
+ violations.add(name + " polled without a subscription");
+ throw new IllegalStateException("Consumer is not subscribed to
any topics or assigned any partitions");
+ }
+ checkSuspended();
+ lastPollStart = dst.now();
+ lastPollReturned = false;
+ rebalanceIfNeeded();
+ // [A] the auto-commit of the positions, after the group was
joined and before the fetch
+ if (autoCommit && dst.now() - lastAutoCommit >=
autoCommitIntervalMs) {
+ autoCommit("interval");
+ }
+ if (faults && dst.chance(pollFailRate)) {
+ dst.block(dst.between(1, 50), name + " poll fails");
+ ledger.record("pollFault", "c", name);
+ throw Dst.injected(new KafkaException("dst: poll failed"));
+ }
+ long waitMillis = Math.max(1, timeout.toMillis());
+ boolean ready = dst.awaitUntil(
+ () -> wakeup || closed || fetchable() || generation !=
FakeKafka.this.generation, waitMillis,
+ name + " poll");
+ if (ready && !wakeup && !closed) {
+ // reads take effect at the end of their latency
+ dst.block(dst.between(1, 20), name + " fetch");
+ }
+ if (closed) {
+ violations.add(name + " closed while it polls");
+ throw new IllegalStateException("This consumer has already
been closed.");
+ }
+ checkWakeup();
+ rebalanceIfNeeded();
+ Map<TopicPartition, List<ConsumerRecord<Object, Object>>> map =
new LinkedHashMap<>();
+ int budget = maxPollRecords;
+ if (budget > 1 && dst.chance(0.3)) {
+ // [P] "fewer records might be returned"
+ budget = (int) dst.between(1, budget - 1);
+ }
+ for (TopicPartition tp : owned) {
+ if (paused.contains(tp) || budget == 0) {
+ continue;
+ }
+ List<Rec> partitionLog = log.get(tp.partition());
+ long pos = position.get(tp);
+ List<ConsumerRecord<Object, Object>> list = new ArrayList<>();
+ while (budget > 0 && pos < partitionLog.size()) {
+ Rec r = partitionLog.get((int) pos);
+ list.add(new ConsumerRecord<>(
+ TOPIC, tp.partition(), r.offset, Dst.EPOCH_MILLIS
+ dst.now(), TimestampType.CREATE_TIME, -1,
+ -1, null, r.id, new RecordHeaders(),
Optional.empty(), Optional.empty()));
+ ledger.record("fetched", "p" + tp.partition(), r.id);
+ pos++;
+ budget--;
+ }
+ position.put(tp, pos);
+ if (!list.isEmpty()) {
+ map.put(tp, list);
+ }
+ }
+ if (!map.isEmpty()) {
+ dst.trace("fake: " + name + " poll returns " +
map.values().stream().mapToInt(List::size).sum()
+ + " records " + positions());
+ }
+ lastPollReturned = true;
+ return new ConsumerRecords<>(map, Map.of());
+ }
+
+ /**
+ * The check of a suspended route: once the consumer completed a poll
that started after the route was suspended
+ * (and then handled its records and its pause request), it must have
paused its partitions.
+ */
+ private void checkSuspended() {
+ if (suspendedSince >= 0 && lastPollStart >= suspendedSince &&
lastPollReturned) {
+ Set<TopicPartition> fetching = new
TreeSet<>(FakeKafka::compare);
+ fetching.addAll(owned);
+ fetching.removeAll(paused);
+ if (!fetching.isEmpty()) {
+ suspendViolations.add(name + " still fetches " + fetching
+ " at t=" + dst.now()
+ + " after the route was suspended at
t=" + suspendedSince);
+ }
+ }
+ }
+
+ private String positions() {
+ return "positions=" + position;
+ }
+
+ private boolean fetchable() {
+ for (TopicPartition tp : owned) {
+ if (!paused.contains(tp) && position.get(tp) <
log.get(tp.partition()).size()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** [G] an eager rebalance inside poll, when the group changed since
this member last joined. */
+ private void rebalanceIfNeeded() {
+ if (generation == FakeKafka.this.generation) {
+ return;
+ }
+ // the join and sync round trips of the group
+ dst.block(dst.between(1, 50), name + " rebalance");
+ if (closed || !subscribed) {
+ return;
+ }
+ if (autoCommit) {
+ autoCommit("before revocation");
+ }
+ if (!owned.isEmpty() && listener != null) {
+ RuntimeException failure = revoke();
+ if (interruption(failure)) {
+ // [L] thrown at once: the member keeps its partitions and
joins again in its next poll
+ throw failure;
+ }
+ if (failure != null) {
+ // [L] the partitions are given up, the member joins again
in its next poll
+ owned.clear();
+ paused.clear();
+ position.clear();
+ throw listenerFailure(failure);
+ }
+ }
+ if (closed || !subscribed) {
+ return;
+ }
+ long gen = FakeKafka.this.generation;
+ Set<TopicPartition> assigned = assignmentOf(this);
+ owned.clear();
+ paused.clear();
+ position.clear();
+ owned.addAll(assigned);
+ for (TopicPartition tp : owned) {
+ Long c = committed.get(tp.partition());
+ position.put(tp, c != null ? c : 0L);
+ }
+ generation = gen;
+ dst.trace("fake: " + name + " assigned " + owned + " (generation "
+ gen + ") " + positions());
+ if (listener != null) {
+ try {
+ listener.onPartitionsAssigned(copyOf(owned));
+ } catch (RuntimeException e) {
+ // [L] the partitions are taken, then poll throws
+ throw listenerFailure(e);
+ }
+ }
+ }
+
+ /** [A] commits the positions (before the fetch of a poll, before a
revocation, at close). */
+ private void autoCommit(String why) {
+ lastAutoCommit = dst.now();
+ if (owned.isEmpty()) {
+ return;
+ }
+ if (faults && dst.chance(commitFailRate)) {
+ dst.trace("fake: " + name + " auto-commit (" + why + ")
fails");
+ ledger.record("commitFault", "c", name);
+ return;
+ }
+ Map<TopicPartition, OffsetAndMetadata> offsets = new
TreeMap<>(FakeKafka::compare);
+ for (TopicPartition tp : owned) {
+ offsets.put(tp, new OffsetAndMetadata(position.get(tp)));
+ }
+ apply(this, offsets, "auto-commit (" + why + ")");
+ }
+
+ @SuppressWarnings("unchecked")
+ private void commitSync(Object[] args) {
+ invokeCallbacks();
+ checkWakeup();
+ Map<TopicPartition, OffsetAndMetadata> offsets = args != null &&
args.length > 0 && args[0] instanceof Map
+ ? (Map<TopicPartition, OffsetAndMetadata>) args[0] :
consumedPositions();
+ long timeout = defaultApiTimeoutMs;
+ if (args != null && args.length > 0 && args[args.length - 1]
instanceof Duration d) {
+ timeout = d.toMillis();
+ }
+ checkCommittable(offsets);
+ long latency = commitLatency();
+ if ((faults && dst.chance(commitFailRate)) || latency > timeout) {
+ // [S] the timeout expires: the broker may or may not have
applied the commit
+ boolean applied = dst.chance(0.5);
+ if (applied) {
+ apply(this, offsets, "commitSync (it times out)");
+ }
+ ledger.record("commitFault", "c", name);
+ dst.block(timeout, name + " commitSync times out" + (applied ?
" (applied)" : " (not applied)"));
+ throw Dst.injected(new TimeoutException(
+ "dst: Timeout of " + timeout + "ms expired before
successfully committing offsets " + offsets));
+ }
+ // writes take effect at the start of their latency
+ apply(this, offsets, "commitSync");
+ dst.block(latency, name + " commitSync");
+ }
+
+ @SuppressWarnings("unchecked")
+ private void commitAsync(Object[] args) {
+ invokeCallbacks();
+ Map<TopicPartition, OffsetAndMetadata> offsets;
+ OffsetCommitCallback callback = null;
+ if (args == null || args.length == 0) {
+ offsets = consumedPositions();
+ } else if (args.length == 1) {
+ offsets = consumedPositions();
+ callback = (OffsetCommitCallback) args[0];
+ } else {
+ offsets = (Map<TopicPartition, OffsetAndMetadata>) args[0];
+ callback = (OffsetCommitCallback) args[1];
+ }
+ Exception error = null;
+ try {
+ checkCommittable(offsets);
+ } catch (CommitFailedException e) {
+ error = e;
+ }
+ if (error == null && faults && dst.chance(commitFailRate)) {
+ ledger.record("commitFault", "c", name);
+ error = Dst.injected(new RetriableCommitFailedException("dst:
async commit failed"));
+ }
+ if (error == null) {
+ apply(this, offsets, "commitAsync");
+ } else {
+ dst.trace("fake: " + name + " commitAsync " + offsets + "
fails: " + error.getMessage());
+ }
+ // [C] the callback runs at a later call of this consumer, in order
+ callbacks.add(new PendingCallback(copyOf(offsets), error,
callback));
+ }
+
+ private void invokeCallbacks() {
+ while (!callbacks.isEmpty()) {
+ PendingCallback c = callbacks.remove(0);
+ if (c.callback() != null) {
+ c.callback().onComplete(c.offsets(), c.error());
+ }
+ }
+ }
+
+ private Map<TopicPartition, OffsetAndMetadata> consumedPositions() {
+ Map<TopicPartition, OffsetAndMetadata> offsets = new
TreeMap<>(FakeKafka::compare);
+ for (TopicPartition tp : owned) {
+ offsets.put(tp, new OffsetAndMetadata(position.get(tp)));
+ }
+ return offsets;
+ }
+
+ /** [S] the group takes a commit of a member for the partitions it
owns. */
+ private void checkCommittable(Map<TopicPartition, OffsetAndMetadata>
offsets) {
+ for (TopicPartition tp : offsets.keySet()) {
+ if (!owned.contains(tp)) {
+ dst.trace("fake: " + name + " commit of " + tp + ", which
it does not own " + offsets);
+ throw new CommitFailedException(
+ "dst: Commit cannot be completed since the group
has already rebalanced and assigned the"
+ + " partitions to another
member");
+ }
+ }
+ }
+
+ private Map<TopicPartition, OffsetAndMetadata> committed(Object[]
args) {
+ checkWakeup();
+ @SuppressWarnings("unchecked")
+ Set<TopicPartition> partitions = (Set<TopicPartition>) args[0];
+ // [R] a remote call: reads take effect at the end of their latency
+ dst.block(dst.chance(0.2) ? dst.between(100, 2000) :
dst.between(1, 30), name + " committed");
+ checkWakeup();
+ Map<TopicPartition, OffsetAndMetadata> answer = new
TreeMap<>(FakeKafka::compare);
+ for (TopicPartition tp : partitions) {
+ Long c = committed.get(tp.partition());
+ answer.put(tp, c != null ? new OffsetAndMetadata(c) : null);
+ }
+ return answer;
+ }
+
+ private void seek(TopicPartition tp, long offset) {
+ if (!owned.contains(tp)) {
+ violations.add(name + " seeks " + tp + ", which is not
assigned");
+ throw new IllegalStateException("No current assignment for
partition " + tp);
+ }
+ if (offset < 0) {
+ violations.add(name + " seeks " + tp + " to " + offset);
+ throw new IllegalArgumentException("seek offset must not be a
negative number");
+ }
+ position.put(tp, offset);
+ dst.trace("fake: " + name + " seek " + tp + " to " + offset);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void pause(Object[] args, boolean pause) {
+ Collection<TopicPartition> partitions =
(Collection<TopicPartition>) args[0];
+ for (TopicPartition tp : partitions) {
+ if (!owned.contains(tp)) {
+ violations.add(name + (pause ? " pauses " : " resumes ") +
tp + ", which is not assigned");
+ throw new IllegalStateException("No current assignment for
partition " + tp);
+ }
+ }
+ if (pause) {
+ paused.addAll(partitions);
+ } else {
+ paused.removeAll(partitions);
+ }
+ dst.trace("fake: " + name + (pause ? " pause " : " resume ") +
partitions);
+ }
+
+ private void close() {
+ if (closed) {
+ return;
+ }
+ dst.trace("fake: " + name + " close");
+ // [L] the classic client turns wakeups off before it closes
+ wakeup = false;
+ try {
+ if (subscribed) {
+ if (autoCommit) {
+ autoCommit("close");
+ }
+ invokeCallbacks();
+ leave();
+ } else {
+ invokeCallbacks();
+ }
+ } finally {
+ // as KafkaConsumer.close: the consumer is closed, then the
first exception (of a callback) is thrown
+ closed = true;
+ open.remove(this);
+ }
+ }
+ }
+
+ private final Dst dst;
+ private final DstLedger ledger;
+ final List<List<Rec>> log = new ArrayList<>();
+ /** The committed offset of each partition of the group (the next record
to read), absent if none. */
+ final Map<Integer, Long> committed = new TreeMap<>();
+ final List<String> violations = new ArrayList<>();
+ final List<String> lossViolations = new ArrayList<>();
+ final List<String> suspendViolations = new ArrayList<>();
+ final List<Member> open = new ArrayList<>();
+ final List<Member> members = new ArrayList<>();
+ final Set<Integer> phantom = new TreeSet<>();
+ long generation;
+ boolean autoCommit;
+ long autoCommitIntervalMs = 5000;
+ long defaultApiTimeoutMs = 60_000;
+ boolean breakOnFirstError;
+ int maxPollRecords = 500;
+ double createFailRate;
+ double pollFailRate;
+ double commitFailRate;
+ boolean slowCommits;
+ boolean faults = true;
+ /** Since when the route is suspended (virtual time), or -1. */
+ long suspendedSince = -1;
+ private int created;
+
+ FakeKafka(Dst dst, DstLedger ledger, int partitions) {
+ this.dst = dst;
+ this.ledger = ledger;
+ for (int p = 0; p < partitions; p++) {
+ log.add(new ArrayList<>());
+ }
+ }
+
+ /** A copy in partition order (TopicPartition is not Comparable). */
+ static Set<TopicPartition> copyOf(Collection<TopicPartition> partitions) {
+ Set<TopicPartition> copy = new TreeSet<>(FakeKafka::compare);
+ copy.addAll(partitions);
+ return copy;
+ }
+
+ static Map<TopicPartition, OffsetAndMetadata> copyOf(Map<TopicPartition,
OffsetAndMetadata> offsets) {
+ Map<TopicPartition, OffsetAndMetadata> copy = new
TreeMap<>(FakeKafka::compare);
+ copy.putAll(offsets);
+ return copy;
+ }
+
+ static int compare(TopicPartition a, TopicPartition b) {
+ int c = a.topic().compareTo(b.topic());
+ return c != 0 ? c : Integer.compare(a.partition(), b.partition());
+ }
+
+ /** KafkaClientFactory.getConsumer. */
+ @SuppressWarnings("unchecked")
+ Consumer<Object, Object> newConsumer() {
+ if (faults && dst.chance(createFailRate)) {
+ ledger.record("createFault", "c", "c" + created);
+ throw Dst.injected(new KafkaException("dst: Failed to construct
kafka consumer"));
+ }
+ Member member = new Member("c" + created++);
+ open.add(member);
+ dst.trace("fake: created " + member.name);
+ return (Consumer<Object, Object>) Proxy.newProxyInstance(
+ Consumer.class.getClassLoader(), new Class<?>[] {
Consumer.class }, member);
+ }
+
+ void publish(int partition, String id) {
+ List<Rec> partitionLog = log.get(partition);
+ partitionLog.add(new Rec(partition, partitionLog.size(), id));
+ ledger.record("published", "p" + partition, id);
+ }
+
+ Rec rec(int partition, long offset) {
+ return log.get(partition).get((int) offset);
+ }
+
+ /**
+ * Whether the route has finished with a record: it completed, or it
failed and the consumer skips it by design
+ * (breakOnFirstError=false: the error handler handled it). With
breakOnFirstError=true a failed record must be
+ * consumed again.
+ */
+ boolean done(Rec r) {
+ return r.completed > 0 || (r.failed > 0 && !breakOnFirstError);
+ }
+
+ boolean done(TopicPartition tp, long offset) {
+ return done(rec(tp.partition(), offset));
+ }
+
+ /** A member joined or left, or the phantom member did: every member
rebalances at its next poll. */
+ void groupChanged(String why) {
+ generation++;
+ dst.trace("fake: group changed (" + why + "), generation " +
generation);
+ }
+
+ void phantomJoins(Set<Integer> partitions) {
+ phantom.clear();
+ phantom.addAll(partitions);
+ groupChanged("a phantom member joins and takes " + partitions);
+ }
+
+ void phantomLeaves() {
+ if (!phantom.isEmpty()) {
+ phantom.clear();
+ groupChanged("the phantom member leaves");
+ }
+ }
+
+ /** The partitions of a member in the current generation: the phantom's
first, the rest in turn. */
+ private Set<TopicPartition> assignmentOf(Member member) {
+ Set<TopicPartition> answer = new TreeSet<>(FakeKafka::compare);
+ int index = members.indexOf(member);
+ if (index < 0) {
+ return answer;
+ }
+ int turn = 0;
+ for (int p = 0; p < log.size(); p++) {
+ if (phantom.contains(p)) {
+ continue;
+ }
+ if (turn % members.size() == index) {
+ answer.add(new TopicPartition(TOPIC, p));
+ }
+ turn++;
+ }
+ return answer;
+ }
+
+ /**
+ * The group takes a commit. A committed offset must not pass a record the
route has not finished with: that record
+ * would not be consumed again by the group (at-least-once).
+ */
+ private void apply(Member member, Map<TopicPartition, OffsetAndMetadata>
offsets, String how) {
+ for (Map.Entry<TopicPartition, OffsetAndMetadata> e :
offsets.entrySet()) {
+ int p = e.getKey().partition();
+ long offset = e.getValue().offset();
+ List<Rec> partitionLog = log.get(p);
+ if (offset > partitionLog.size()) {
+ violations.add(member.name + " committed " + offset + " on p"
+ p + " past the end " + partitionLog.size());
+ }
+ for (long o = 0; o < Math.min(offset, partitionLog.size()); o++) {
+ Rec r = partitionLog.get((int) o);
+ if (!done(r)) {
+ lossViolations.add(member.name + " " + how + " committed
offset " + offset + " on p" + p + " at t="
+ + dst.now() + " but the route has not
finished with " + r.id + " (offset " + o
+ + ", routed " + r.routed + ", completed
" + r.completed + ", failed "
+ + r.failed + ")");
+ break;
+ }
+ }
+ committed.put(p, offset);
+ ledger.record("committed", "p" + p, String.valueOf(offset));
+ }
+ dst.trace("fake: " + member.name + " " + how + " " + offsets);
+ }
+
+ private long commitLatency() {
+ return faults && slowCommits && dst.chance(0.2) ? dst.between(100,
3000) : dst.between(1, 30);
+ }
+
+ /** Every partition has its committed offset at the end of its log. */
+ boolean allCommitted() {
+ for (int p = 0; p < log.size(); p++) {
+ long c = committed.getOrDefault(p, 0L);
+ if (c < log.get(p).size()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ String notCommitted() {
+ StringBuilder sb = new StringBuilder();
+ for (int p = 0; p < log.size(); p++) {
+ sb.append("p").append(p).append("
committed=").append(committed.get(p)).append(" end=")
+ .append(log.get(p).size()).append(' ');
+ }
+ return sb.toString().trim();
+ }
+
+ String openNames() {
+ List<String> names = new ArrayList<>();
+ for (Member m : open) {
+ names.add(m.name);
+ }
+ return names.toString();
+ }
+}
diff --git
a/components/camel-kafka/src/test/java/org/apache/camel/component/kafka/dst/KafkaConsumerDstTest.java
b/components/camel-kafka/src/test/java/org/apache/camel/component/kafka/dst/KafkaConsumerDstTest.java
new file mode 100644
index 000000000000..568359beca97
--- /dev/null
+++
b/components/camel-kafka/src/test/java/org/apache/camel/component/kafka/dst/KafkaConsumerDstTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.kafka.dst;
+
+import org.apache.camel.test.dst.Dst;
+import org.apache.camel.test.dst.DstInstrument;
+import org.apache.camel.test.dst.DstTest;
+
+/**
+ * Deterministic simulation test of the camel-kafka consumer against a fake
Kafka cluster and consumer group, with
+ * consumer creation failures, poll failures, failed and slow commits, failed
and slow processing, route restarts,
+ * suspend and resume of the route, and rebalances: a committed offset never
passes a record the route has not finished
+ * with, a suspended route stops fetching, and once the faults cleared every
record is finished and committed. A failing
+ * seed is replayed with {@code -Ddst.seed=<seed>}; {@code -Ddst.seeds=5000}
runs more seeds;
+ * {@code -Ddst.kafka.modes=BATCH} and {@code
-Ddst.kafka.pollOnError=RECONNECT} narrow the configurations.
+ */
+@DstInstrument(value = "org.apache.camel.component.kafka",
+ alsoRewrite = {
+ "org.apache.camel.util.StopWatch",
"org.apache.camel.impl.engine.DefaultShutdownStrategy",
+ "org.apache.camel.support.task" })
+class KafkaConsumerDstTest {
+
+ @DstTest(seeds = 1000)
+ void committedOffsetsNeverPassAnUnfinishedRecord(Dst dst) throws Exception
{
+ new KafkaWorld(dst).run();
+ }
+}
diff --git
a/components/camel-kafka/src/test/java/org/apache/camel/component/kafka/dst/KafkaWorld.java
b/components/camel-kafka/src/test/java/org/apache/camel/component/kafka/dst/KafkaWorld.java
new file mode 100644
index 000000000000..6a0bb45807c8
--- /dev/null
+++
b/components/camel-kafka/src/test/java/org/apache/camel/component/kafka/dst/KafkaWorld.java
@@ -0,0 +1,557 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.kafka.dst;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.camel.Exchange;
+import org.apache.camel.Message;
+import org.apache.camel.ServiceStatus;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.kafka.KafkaClientFactory;
+import org.apache.camel.component.kafka.KafkaComponent;
+import org.apache.camel.component.kafka.KafkaConfiguration;
+import org.apache.camel.component.kafka.KafkaConstants;
+import
org.apache.camel.component.kafka.consumer.DefaultKafkaManualAsyncCommitFactory;
+import
org.apache.camel.component.kafka.consumer.DefaultKafkaManualCommitFactory;
+import org.apache.camel.component.kafka.consumer.KafkaManualCommit;
+import org.apache.camel.impl.DefaultCamelContext;
+import org.apache.camel.test.dst.Dst;
+import org.apache.camel.test.dst.DstCamel;
+import org.apache.camel.test.dst.DstExecutor;
+import org.apache.camel.test.dst.DstLedger;
+import org.apache.camel.test.dst.DstWorld;
+import org.apache.camel.test.dst.Invariants;
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.producer.Producer;
+
+/**
+ * The world around a camel-kafka consumer: a fake Kafka cluster and consumer
group ({@link FakeKafka}), a producer, a
+ * route whose processing is sometimes slow and sometimes fails, and an
operator that restarts the route, suspends and
+ * resumes it (with a second suspend and resume while the consumer still
handles the first), and lets a phantom member
+ * join and leave the group, so that partitions are revoked and assigned again
while records are processed. Faults: the
+ * consumer cannot be created, polls fail, commits fail or time out, commits
are slow, processing fails.
+ * <p>
+ * The seed picks the commit mode: Kafka's auto-commit
(autoCommitEnable=true), manual synchronous or asynchronous
+ * commits in the route (allowManualCommit=true with
DefaultKafkaManualCommitFactory or
+ * DefaultKafkaManualAsyncCommitFactory, after every record or only for the
last record of a poll), or batching
+ * (batching=true with autoCommitEnable=false: Camel commits each batch once
it was processed), and breakOnFirstError
+ * and pollOnError (ERROR_HANDLER, DISCARD, RETRY, RECONNECT; not STOP, which
ends the consumer by design).
+ * <p>
+ * Not explored: batching with Kafka's auto-commit (the auto-commit commits
the position of records that Camel still
+ * holds in an incomplete batch, which the KafkaConsumer Javadoc warns about),
and batching with breakOnFirstError (the
+ * failed batch is committed, see "Break on First Error in Batching Mode" in
the component documentation); manual
+ * commits from another thread; consumersCount above 1; seekTo, offset
repositories and resume strategies.
+ * <p>
+ * Left out of the default set: breakOnFirstError with the synchronous manual
commits (SyncCommitManager), where a
+ * commitSync that times out in the break path is being looked at separately;
+ * {@code -Ddst.kafka.syncBreakOnFirstError=true} puts the combination back.
+ * <p>
+ * consumersCount is 1: a consumer thread loops for ever, and with the
one-stack model a second looping thread nested
+ * above the first would never let it resume. For the same reason a route stop
that runs while the consumer thread waits
+ * in poll cannot see it finish: the stop wakes up the Kafka consumer, waits
for its timeouts, and the thread ends after
+ * the stop returned, so the operator starts the route again only once the
previous Kafka consumer is closed. The
+ * operator does not stop or suspend the route while an exchange is in the
route (the shutdown strategy would wait for
+ * it for ever, as it cannot see it finish) or while the consumer thread waits
for the background task that creates its
+ * Kafka consumer (a stop interrupts that wait with real threads, which the
one-stack model does not do). The lock of
+ * KafkaFetchRecords does not exclude on the one stack, so a stop never waits
for the processing of the current records.
+ * <p>
+ * Properties: a committed offset never passes a record the route has not
finished with (at-least-once: completed, or
+ * failed with breakOnFirstError=false, which skips it by design); a suspended
route stops fetching once its consumer
+ * handled the suspend; once the faults cleared, every record is finished, the
committed offsets reach the end of each
+ * partition, and the route holds one open Kafka consumer; the consumer uses
the client as the Javadoc requires; no
+ * Kafka consumer is left open after the route stopped. Records may be
consumed more than once.
+ */
+@DstWorld(component = "kafka",
+ fake = "FakeKafka: a topic with one or two partitions and a consumer
group (classic protocol, eager"
+ + " rebalance), behind the Consumer interface of
KafkaClientFactory",
+ covers = {
+ "commits by Kafka's auto-commit, by manual synchronous or
asynchronous commits in the route, and per"
+ + " batch with batching=true",
+ "breakOnFirstError",
+ "pollOnError ERROR_HANDLER, DISCARD, RETRY and RECONNECT",
+ "a poll that fails while a batch is incomplete",
+ "route restart, also while the consumer polls",
+ "route suspend and resume, with a second request while the
consumer handles the first",
+ "partitions revoked and assigned again by a rebalance while
records are processed" },
+ faults = {
+ "the Kafka consumer cannot be created", "poll fails",
"commitSync times out",
+ "commitAsync fails", "auto-commit fails", "commits are
slow", "committed answers slowly",
+ "processing fails", "processing is slow", "a member joins
and leaves the group" },
+ invariants = {
+ "the consumer uses the Kafka consumer as documented",
+ "a committed offset never passes a record the route has not
finished with",
+ "a suspended route stops fetching",
+ "every record is finished once the faults cleared",
+ "the committed offsets reach the end once the faults
cleared",
+ "the route holds one Kafka consumer once the faults cleared",
+ "no Kafka consumer is left open after the route stopped" },
+ issues = { "CAMEL-25313", "CAMEL-25756" })
+final class KafkaWorld {
+
+ static final String ROUTE = "kafka";
+
+ private final Dst dst;
+ private final DstLedger ledger;
+ private final FakeKafka kafka;
+ private final int partitions;
+ private final String mode;
+ private final boolean breakOnFirstError;
+ private final boolean commitEveryRecord;
+ private final String pollOnError;
+ private final long pollTimeoutMs;
+ private final long shutdownTimeout;
+ private final long commitTimeoutMs;
+ private final long backoffInterval;
+ private final double processFailRate;
+ private final List<String> violations = new ArrayList<>();
+ private boolean faults = true;
+ private int sendsLeft;
+ private int opsLeft;
+ // exchanges in the route: a route stop or suspend nested above one on the
one stack would wait for it for ever
+ // (the shutdown strategy cannot see it finish), so the operator stops and
suspends only when none is in the route
+ private int inRoute;
+ private boolean finished;
+ private String finalStatus;
+ private String finalOpen;
+ private int finalOpenCount;
+ private boolean finalDone;
+ private boolean finalCommitted;
+
+ KafkaWorld(Dst dst) {
+ this.dst = dst;
+ this.ledger = new DstLedger(dst);
+ this.partitions = dst.pick(List.of(1, 2));
+ this.kafka = new FakeKafka(dst, ledger, partitions);
+ this.mode =
dst.pick(Arrays.asList(System.getProperty("dst.kafka.modes",
"AUTO,SYNC,ASYNC,BATCH").split(",")));
+ // see the class Javadoc: batching with breakOnFirstError is not
explored, and the synchronous manual commits
+ // with breakOnFirstError are left out unless
-Ddst.kafka.syncBreakOnFirstError=true
+ this.breakOnFirstError = !"BATCH".equals(mode)
+ && (!"SYNC".equals(mode) ||
Boolean.getBoolean("dst.kafka.syncBreakOnFirstError"))
+ && dst.chance(0.5);
+ this.commitEveryRecord = dst.chance(0.5);
+ this.pollOnError = dst.pick(Arrays.asList(
+ System.getProperty("dst.kafka.pollOnError",
"ERROR_HANDLER,DISCARD,RETRY,RECONNECT").split(",")));
+ this.pollTimeoutMs = dst.pick(List.of(100L, 1000L, 5000L));
+ this.shutdownTimeout = dst.pick(List.of(1000L, 30000L));
+ this.commitTimeoutMs = dst.pick(List.of(1000L, 5000L));
+ this.backoffInterval = dst.pick(List.of(500L, 5000L));
+ kafka.autoCommit = "AUTO".equals(mode);
+ kafka.autoCommitIntervalMs = dst.pick(List.of(1000L, 5000L));
+ kafka.breakOnFirstError = breakOnFirstError;
+ kafka.maxPollRecords = "BATCH".equals(mode) ? dst.pick(List.of(3, 10))
: dst.pick(List.of(1, 3, 10));
+ kafka.createFailRate = dst.pick(List.of(0.0, 0.1, 0.3));
+ kafka.pollFailRate = dst.pick(List.of(0.0, 0.02, 0.1));
+ kafka.commitFailRate = dst.pick(List.of(0.0, 0.05, 0.2));
+ kafka.slowCommits = dst.chance(0.5);
+ this.processFailRate = dst.pick(List.of(0.0, 0.05, 0.2));
+ dst.trace("world: " + config());
+ }
+
+ private String config() {
+ return "partitions=" + partitions + " mode=" + mode + "
breakOnFirstError=" + breakOnFirstError
+ + " commitEveryRecord=" + commitEveryRecord + " pollOnError=" +
pollOnError + " pollTimeoutMs="
+ + pollTimeoutMs + " maxPollRecords=" + kafka.maxPollRecords + "
shutdownTimeout=" + shutdownTimeout
+ + " commitTimeoutMs=" + commitTimeoutMs + "
autoCommitIntervalMs=" + kafka.autoCommitIntervalMs
+ + " backoff=" + backoffInterval + " createFail=" +
kafka.createFailRate + " pollFail="
+ + kafka.pollFailRate + " commitFail=" + kafka.commitFailRate +
" slowCommits=" + kafka.slowCommits
+ + " processFail=" + processFailRate;
+ }
+
+ void run() throws Exception {
+ DefaultCamelContext context = DstCamel.newContext(dst);
+ KafkaComponent component = new KafkaComponent();
+ component.setCreateConsumerBackoffInterval(backoffInterval);
+ if ("SYNC".equals(mode)) {
+ component.setKafkaManualCommitFactory(new
DefaultKafkaManualCommitFactory());
+ } else if ("ASYNC".equals(mode)) {
+ component.setKafkaManualCommitFactory(new
DefaultKafkaManualAsyncCommitFactory());
+ }
+ component.setKafkaClientFactory(new KafkaClientFactory() {
+ @Override
+ public Producer getProducer(Properties kafkaProps) {
+ throw new UnsupportedOperationException("the world has no
producer endpoint");
+ }
+
+ @Override
+ public Consumer getConsumer(Properties kafkaProps) {
+ return kafka.newConsumer();
+ }
+
+ @Override
+ public String getBrokers(KafkaConfiguration configuration) {
+ return configuration.getBrokers();
+ }
+ });
+ context.addComponent("kafka", component);
+ boolean manual = "SYNC".equals(mode) || "ASYNC".equals(mode);
+ String uri = "kafka:" + FakeKafka.TOPIC +
"?brokers=127.0.0.1:9092&groupId=" + FakeKafka.GROUP
+ +
"&consumersCount=1&autoOffsetReset=earliest&pollTimeoutMs=" + pollTimeoutMs +
"&maxPollRecords="
+ + kafka.maxPollRecords + "&shutdownTimeout=" +
shutdownTimeout + "&commitTimeoutMs="
+ + commitTimeoutMs + "&pollOnError=" + pollOnError +
"&breakOnFirstError=" + breakOnFirstError
+ + "&autoCommitEnable=" + "AUTO".equals(mode) +
"&autoCommitIntervalMs="
+ + kafka.autoCommitIntervalMs + "&allowManualCommit=" +
manual + "&batching=" + "BATCH".equals(mode);
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+ from(uri).routeId(ROUTE).autoStartup(false)
+ .process(exchange -> process(exchange, manual));
+ }
+ });
+ context.start();
+ DstExecutor operator = dst.executor("operator", 1);
+ try {
+ operator.schedule("start route", 0, () -> startRoute(context,
operator));
+ planSends();
+ planOperations(context, operator);
+ operator.schedule("finisher", 1000, () -> finisher(context,
operator, -1));
+ dst.runUntil(() -> finished, "the finisher");
+ // the consumer thread ends after the stop; let it close its Kafka
consumer
+ dst.runFor(2 * shutdownTimeout + 10_000);
+ dst.check(
+ Invariants.of("the consumer uses the Kafka consumer as
documented",
+ () -> kafka.violations.isEmpty() &&
violations.isEmpty()
+ ? null : kafka.violations + " " +
violations + " (" + config() + ")"),
+ Invariants.of("a committed offset never passes a record
the route has not finished with",
+ () -> kafka.lossViolations.isEmpty() ? null :
kafka.lossViolations + " (" + config() + ")"),
+ Invariants.of("a suspended route stops fetching",
+ () -> kafka.suspendViolations.isEmpty()
+ ? null : kafka.suspendViolations + " (" +
config() + ")"),
+ Invariants.of("every record is finished once the faults
cleared",
+ () -> finalDone ? null : "not finished: " +
notDone() + " (" + config() + ")"),
+ Invariants.of("the committed offsets reach the end once
the faults cleared",
+ () -> finalCommitted ? null : kafka.notCommitted()
+ " (" + config() + ")"),
+ Invariants.of("the route holds one Kafka consumer once the
faults cleared",
+ () -> "Started".equals(finalStatus) &&
finalOpenCount == 1
+ ? null
+ : "route is " + finalStatus + ", open
Kafka consumers " + finalOpen + " ("
+ + config() + ")"),
+ Invariants.of("no Kafka consumer is left open after the
route stopped",
+ () -> kafka.open.isEmpty()
+ ? null : "open after stop: " +
kafka.openNames() + " (" + config() + ")"));
+ } finally {
+ dst.dropPending();
+ context.stop();
+ }
+ }
+
+ /** The route: a record (or each record of a batch) is routed, maybe
slowly, then fails or completes. */
+ private void process(Exchange exchange, boolean manual) throws Exception {
+ List<FakeKafka.Rec> recs = new ArrayList<>();
+ Object body = exchange.getMessage().getBody();
+ if (!(body instanceof List) &&
exchange.getMessage().getHeader(KafkaConstants.OFFSET) == null) {
+ // not a record: an exception of the consumer bridged to the error
handler
+ return;
+ }
+ if (body instanceof List<?> batch) {
+ for (Object o : batch) {
+ recs.add(recOf(((Exchange) o).getMessage()));
+ }
+ } else {
+ recs.add(recOf(exchange.getMessage()));
+ }
+ inRoute++;
+ try {
+ processRecords(exchange, recs, manual);
+ } finally {
+ inRoute--;
+ }
+ }
+
+ private void processRecords(Exchange exchange, List<FakeKafka.Rec> recs,
boolean manual) {
+ String ids = recs.stream().map(r -> r.id).toList().toString();
+ for (FakeKafka.Rec r : recs) {
+ r.routed++;
+ ledger.record("routed", "p" + r.partition, r.id);
+ }
+ if (faults && dst.chance(0.1)) {
+ dst.block(dst.between(1000, 10_000), "slow processing of " + ids);
+ } else {
+ dst.block(dst.between(0, 100), "processing of " + ids);
+ }
+ if (faults && dst.chance(processFailRate)) {
+ for (FakeKafka.Rec r : recs) {
+ r.failed++;
+ ledger.record("failed", "p" + r.partition, r.id);
+ }
+ throw Dst.injected(new IllegalStateException("dst: processing
failed for " + ids));
+ }
+ for (FakeKafka.Rec r : recs) {
+ r.completed++;
+ ledger.record("completed", "p" + r.partition, r.id);
+ }
+ if (manual) {
+ Message m = exchange.getMessage();
+ boolean last =
Boolean.TRUE.equals(m.getHeader(KafkaConstants.LAST_RECORD_BEFORE_COMMIT,
Boolean.class));
+ if (commitEveryRecord || last) {
+ // a commit that fails fails the exchange, after the record
was processed
+ m.getHeader(KafkaConstants.MANUAL_COMMIT,
KafkaManualCommit.class).commit();
+ }
+ }
+ }
+
+ private FakeKafka.Rec recOf(Message message) {
+ int p = message.getHeader(KafkaConstants.PARTITION, Integer.class);
+ long offset = message.getHeader(KafkaConstants.OFFSET, Long.class);
+ FakeKafka.Rec r = kafka.rec(p, offset);
+ if (!r.id.equals(message.getBody(String.class))) {
+ violations.add("record p" + p + "@" + offset + " has the body " +
message.getBody() + ", not " + r.id);
+ }
+ return r;
+ }
+
+ private String notDone() {
+ List<String> missing = new ArrayList<>();
+ for (List<FakeKafka.Rec> partitionLog : kafka.log) {
+ for (FakeKafka.Rec r : partitionLog) {
+ if (!kafka.done(r)) {
+ missing.add(r.id + "@p" + r.partition + "/" + r.offset + "
routed=" + r.routed + " failed=" + r.failed);
+ }
+ }
+ }
+ return missing.toString();
+ }
+
+ private boolean allDone() {
+ for (List<FakeKafka.Rec> partitionLog : kafka.log) {
+ for (FakeKafka.Rec r : partitionLog) {
+ if (!kafka.done(r)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Waits for the sends and operations, switches the faults off, publishes
one more record to each partition (its
+ * commit covers the records before it), waits for everything to be
finished and committed (or a deadline), takes
+ * the final state and stops the route.
+ */
+ private void finisher(DefaultCamelContext context, DstExecutor operator,
long deadline) {
+ if (deadline == -1 && (sendsLeft > 0 || opsLeft > 0)) {
+ operator.schedule("finisher", 500, () -> finisher(context,
operator, -1));
+ return;
+ }
+ if (deadline == -1) {
+ long slow = dst.between(0, 10_000);
+ dst.trace("world: operations done, faults for another " + slow + "
ms");
+ operator.schedule("finisher", slow, () -> finisher(context,
operator, -2));
+ return;
+ }
+ if (deadline == -2) {
+ faults = false;
+ kafka.faults = false;
+ kafka.phantomLeaves();
+ for (int p = 0; p < partitions; p++) {
+ kafka.publish(p, "end" + p);
+ }
+ dst.trace("world: faults cleared");
+ long d = dst.now() + 10 * 60_000L;
+ operator.schedule("finisher", 500, () -> finisher(context,
operator, d));
+ return;
+ }
+ ServiceStatus status =
context.getRouteController().getRouteStatus(ROUTE);
+ boolean done = allDone();
+ boolean committed = kafka.allCommitted();
+ if ((inRoute > 0 || !done || !committed || status !=
ServiceStatus.Started || kafka.open.size() != 1)
+ && dst.now() < deadline) {
+ operator.schedule("finisher", 500, () -> finisher(context,
operator, deadline));
+ return;
+ }
+ finalStatus = String.valueOf(status);
+ finalDone = done;
+ finalCommitted = committed;
+ finalOpenCount = kafka.open.size();
+ finalOpen = kafka.openNames();
+ if (inRoute > 0) {
+ // a stop would wait for it for ever (one-stack model)
+ finalDone = false;
+ finished = true;
+ return;
+ }
+ try {
+ context.getRouteController().stopRoute(ROUTE);
+ } catch (Exception e) {
+ violations.add("final stop failed: " + e);
+ }
+ finished = true;
+ }
+
+ /** The operator starts the route; a failed start is retried, as the
supervising route controller does. */
+ private void startRoute(DefaultCamelContext context, DstExecutor operator)
{
+ try {
+ context.getRouteController().startRoute(ROUTE);
+ ledger.record("routeStarted", "r", ROUTE);
+ } catch (Exception e) {
+ ledger.record("routeStartFailed", "r", ROUTE);
+ dst.trace("world: start failed (" + e.getMessage() + "), retry in
2 s");
+ operator.schedule("retry start route", 2000, () ->
startRoute(context, operator));
+ }
+ }
+
+ /**
+ * Starts the route again once the Kafka consumer of the stopped route is
closed: its thread ends only after the
+ * stop returned (one-stack model), and a new consumer thread started
above it would never let it resume.
+ */
+ private void startWhenClosed(DefaultCamelContext context, DstExecutor
operator, long since) {
+ if (!kafka.open.isEmpty() && dst.now() - since < 10 * 60_000L) {
+ operator.schedule("start route when the consumer closed", 200,
+ () -> startWhenClosed(context, operator, since));
+ return;
+ }
+ if (!kafka.open.isEmpty()) {
+ violations.add("Kafka consumers " + kafka.openNames() + " still
open 10 minutes after the route stopped");
+ }
+ startRoute(context, operator);
+ opsLeft--;
+ }
+
+ private void planSends() {
+ DstExecutor producer = dst.executor("producer", 1);
+ long t = dst.between(0, 500);
+ int n = 0;
+ int bursts = (int) dst.between(1, 4);
+ for (int b = 0; b < bursts; b++) {
+ int size = (int) dst.between(1, 15);
+ for (int i = 0; i < size; i++) {
+ String id = "m" + n++;
+ int p = partitions == 1 ? 0 : dst.nextInt(partitions);
+ sendsLeft++;
+ producer.schedule("publish " + id, t, () -> {
+ kafka.publish(p, id);
+ sendsLeft--;
+ });
+ t += dst.between(0, 100);
+ }
+ t += dst.between(0, 8000);
+ }
+ }
+
+ private void planOperations(DefaultCamelContext context, DstExecutor
operator) {
+ int ops = (int) dst.between(0, 4);
+ long t = dst.between(500, 3000);
+ for (int i = 0; i < ops; i++) {
+ int kind = dst.nextInt(3);
+ opsLeft++;
+ if (kind == 0) {
+ operator.schedule("stop route", t, () ->
whenConnected(operator, "stop route", () -> {
+ if (context.getRouteController().getRouteStatus(ROUTE) !=
ServiceStatus.Started) {
+ opsLeft--;
+ return;
+ }
+ try {
+ context.getRouteController().stopRoute(ROUTE);
+ ledger.record("routeStopped", "r", ROUTE);
+ } catch (Exception e) {
+ violations.add("stop failed: " + e);
+ }
+ long since = dst.now();
+ operator.schedule("start route when the consumer closed",
dst.between(0, 3000),
+ () -> startWhenClosed(context, operator, since));
+ }));
+ } else if (kind == 1) {
+ // a second suspend and resume while the consumer may still
handle the first resume (it reads the
+ // committed offsets before it resumes its partitions)
+ int times = dst.chance(0.5) ? 2 : 1;
+ operator.schedule("suspend route", t, () ->
suspendAndResume(context, operator, times));
+ } else {
+ Set<Integer> taken = partitions == 1 ? Set.of(0) :
dst.pick(List.of(Set.of(0), Set.of(1), Set.of(0, 1)));
+ long held = dst.between(500, 15_000);
+ operator.schedule("a member joins the group", t, () -> {
+ kafka.phantomJoins(taken);
+ operator.schedule("the member leaves the group", held, ()
-> {
+ kafka.phantomLeaves();
+ opsLeft--;
+ });
+ });
+ }
+ t += dst.between(500, 10_000);
+ }
+ }
+
+ /**
+ * Runs an operation of the operator once no exchange is in the route. The
operator task can run nested above an
+ * exchange that blocks (on the one stack), so it waits in a new task, not
in this one.
+ */
+ private void whenIdle(DstExecutor operator, String what, Runnable
operation) {
+ if (inRoute > 0) {
+ operator.schedule(what + " (an exchange is in the route)", 50, ()
-> whenIdle(operator, what, operation));
+ return;
+ }
+ operation.run();
+ }
+
+ /**
+ * Stops the route only while the consumer thread holds a Kafka consumer,
not while it waits for the background task
+ * that creates one: a stop interrupts that wait with real threads
(shutdownNow), which the one-stack model does not
+ * do, and the consumer thread would go on after the route was started
again.
+ */
+ private void whenConnected(DstExecutor operator, String what, Runnable
operation) {
+ if (kafka.open.size() != 1 || inRoute > 0) {
+ operator.schedule(what + " (no Kafka consumer, or an exchange is
in the route)", 100,
+ () -> whenConnected(operator, what, operation));
+ return;
+ }
+ operation.run();
+ }
+
+ private void suspendAndResume(DefaultCamelContext context, DstExecutor
operator, int times) {
+ whenIdle(operator, "suspend route", () -> {
+ if (context.getRouteController().getRouteStatus(ROUTE) !=
ServiceStatus.Started) {
+ opsLeft--;
+ return;
+ }
+ try {
+ suspend(context);
+ } catch (Exception e) {
+ violations.add("suspend failed: " + e);
+ opsLeft--;
+ return;
+ }
+ operator.schedule("resume route", dst.between(100, 5000), () -> {
+ try {
+ resume(context);
+ } catch (Exception e) {
+ violations.add("resume failed: " + e);
+ }
+ if (times > 1) {
+ operator.schedule("suspend route again", dst.between(0,
pollTimeoutMs + 2000),
+ () -> suspendAndResume(context, operator, times -
1));
+ } else {
+ opsLeft--;
+ }
+ });
+ });
+ }
+
+ private void suspend(DefaultCamelContext context) throws Exception {
+ context.getRouteController().suspendRoute(ROUTE);
+ kafka.suspendedSince = dst.now();
+ ledger.record("routeSuspended", "r", ROUTE);
+ }
+
+ private void resume(DefaultCamelContext context) throws Exception {
+ kafka.suspendedSince = -1;
+ context.getRouteController().resumeRoute(ROUTE);
+ ledger.record("routeResumed", "r", ROUTE);
+ }
+}
diff --git a/components/camel-kafka/src/test/resources/log4j2.properties
b/components/camel-kafka/src/test/resources/log4j2.properties
index 452b67149cc4..45492e1685b2 100644
--- a/components/camel-kafka/src/test/resources/log4j2.properties
+++ b/components/camel-kafka/src/test/resources/log4j2.properties
@@ -15,9 +15,16 @@
## limitations under the License.
## ---------------------------------------------------------------------------
-appender.out.type = File
+# capped: the DST test runs the consumer for a thousand seeds
+appender.out.type = RollingFile
appender.out.name = out
appender.out.fileName = target/camel-kafka-test.log
+appender.out.filePattern = target/camel-kafka-test.%i.log
+appender.out.policies.type = Policies
+appender.out.policies.size.type = SizeBasedTriggeringPolicy
+appender.out.policies.size.size = 50MB
+appender.out.strategy.type = DefaultRolloverStrategy
+appender.out.strategy.max = 1
appender.out.layout.type = PatternLayout
appender.out.layout.pattern = %d [%-15.15t] %-5p %-30.30c{1} - %m%n
appender.stdout.type = Console
diff --git a/components/camel-test/camel-test-dst/src/main/docs/test-dst.adoc
b/components/camel-test/camel-test-dst/src/main/docs/test-dst.adoc
index 0fe66cfbe03a..4923ed2b2935 100644
--- a/components/camel-test/camel-test-dst/src/main/docs/test-dst.adoc
+++ b/components/camel-test/camel-test-dst/src/main/docs/test-dst.adoc
@@ -101,6 +101,18 @@ a|* synchronousPull with ackMode AUTO, and ackMode NONE
with grouped messages ac
* a stop and start of the same consumer in one step, as a throttling route
policy does
|CAMEL-25712, CAMEL-25790, CAMEL-26647
+|camel-kafka
+|`KafkaWorld` (`KafkaConsumerDstTest`)
+|FakeKafka: a topic with one or two partitions and a consumer group (classic
protocol, eager rebalance), behind the Consumer interface of KafkaClientFactory
+a|* commits by Kafka's auto-commit, by manual synchronous or asynchronous
commits in the route, and per batch with batching=true
+* breakOnFirstError
+* pollOnError ERROR_HANDLER, DISCARD, RETRY and RECONNECT
+* a poll that fails while a batch is incomplete
+* route restart, also while the consumer polls
+* route suspend and resume, with a second request while the consumer handles
the first
+* partitions revoked and assigned again by a rebalance while records are
processed
+|CAMEL-25313, CAMEL-25756
+
|camel-kafka-share
|`KafkaShareWorld` (`KafkaShareConsumerDstTest`)
|FakeShareGroup: a share group (KIP-932) of one topic with one partition,
behind ShareConsumer