This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new dc73c347e3c0 CAMEL-26789: camel-chaos - an experiment fails when its
rules lost messages at the dead letter channel (#29512)
dc73c347e3c0 is described below
commit dc73c347e3c0a836b9736019654856a3c31ad447
Author: Claus Ibsen <[email protected]>
AuthorDate: Mon Nov 9 23:50:36 2026 +0100
CAMEL-26789: camel-chaos - an experiment fails when its rules lost messages
at the dead letter channel (#29512)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../org/apache/camel/catalog/docs/chaos.adoc | 5 +++-
components/camel-chaos/src/main/docs/chaos.adoc | 5 +++-
.../camel/component/chaos/ChaosExperiment.java | 27 ++++++++++++++++++----
.../camel/component/chaos/ChaosExperimentTest.java | 26 +++++++++++++++++++++
.../dsl/jbang/core/commands/process/ListChaos.java | 5 ++++
.../jbang/core/commands/process/ListChaosTest.java | 5 +++-
6 files changed, 66 insertions(+), 7 deletions(-)
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/chaos.adoc
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/chaos.adoc
index 66d44391258b..6713e09b3b2d 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/chaos.adoc
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/chaos.adoc
@@ -387,7 +387,10 @@ runs.
When it ends, camel-chaos reads what came of it and then removes its rules:
per route the difference of the counts, the
faults of each rule, the state of each expectation, and a verdict:
-* `FAIL`: an expectation is unmet.
+* `FAIL`: an expectation is unmet, or a rule of the experiment failed or
dropped the send of a dead letter channel.
+The dead letter channel then handles the message without delivering it, so no
expectation of a route sees it: the
+experiment counts it as lost (`lostAtDeadLetterChannel`, and per rule
`lostAtDeadLetterChannelPerRule`), as the
+report does.
* `PASS`: all expectations are met.
* `PENDING`: it is still open and an expectation is not decided yet.
* `INCONCLUSIVE`: it ended before an expectation was decided, so the messages
may still have come.
diff --git a/components/camel-chaos/src/main/docs/chaos.adoc
b/components/camel-chaos/src/main/docs/chaos.adoc
index 66d44391258b..6713e09b3b2d 100644
--- a/components/camel-chaos/src/main/docs/chaos.adoc
+++ b/components/camel-chaos/src/main/docs/chaos.adoc
@@ -387,7 +387,10 @@ runs.
When it ends, camel-chaos reads what came of it and then removes its rules:
per route the difference of the counts, the
faults of each rule, the state of each expectation, and a verdict:
-* `FAIL`: an expectation is unmet.
+* `FAIL`: an expectation is unmet, or a rule of the experiment failed or
dropped the send of a dead letter channel.
+The dead letter channel then handles the message without delivering it, so no
expectation of a route sees it: the
+experiment counts it as lost (`lostAtDeadLetterChannel`, and per rule
`lostAtDeadLetterChannelPerRule`), as the
+report does.
* `PASS`: all expectations are met.
* `PENDING`: it is still open and an expectation is not decided yet.
* `INCONCLUSIVE`: it ended before an expectation was decided, so the messages
may still have come.
diff --git
a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosExperiment.java
b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosExperiment.java
index fe2a5bd5c87b..33f6d676dc53 100644
---
a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosExperiment.java
+++
b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosExperiment.java
@@ -33,9 +33,11 @@ import org.apache.camel.util.json.JsonObject;
* the difference of the counts, the faults of each rule and the states of the
expectations are kept in the journal,
* with a verdict.
* <p/>
- * The verdict is {@code FAIL} when an expectation is unmet and {@code PASS}
when all are met. While it runs and an
- * expectation is not decided it is {@code PENDING}, and an experiment that
ended before an expectation was decided is
- * {@code INCONCLUSIVE}: the messages may still have come. The manager guards
it with its lock.
+ * The verdict is {@code FAIL} when an expectation is unmet, or a rule of it
failed or dropped the send of a dead letter
+ * channel, which then handled the message without delivering it (lost, as the
report counts it, CAMEL-26789), and
+ * {@code PASS} when all are met. While it runs and an expectation is not
decided it is {@code PENDING}, and an
+ * experiment that ended before an expectation was decided is {@code
INCONCLUSIVE}: the messages may still have come.
+ * The manager guards it with its lock.
*/
final class ChaosExperiment {
@@ -135,6 +137,9 @@ final class ChaosExperiment {
jo.put("rules", new JsonArray(rules));
boolean unmet = false;
boolean undecided = false;
+ // the messages its rules kept from a dead letter channel: handled
there, so no expectation of a route sees them
+ long lost = 0;
+ JsonObject lostAtDlq = new JsonObject();
JsonArray texts = new JsonArray();
JsonObject faults = new JsonObject();
JsonArray expectations = new JsonArray();
@@ -149,6 +154,11 @@ final class ChaosExperiment {
fired += r.firedAt(hook);
}
faults.put(id, fired);
+ long kept = r.lostAtDlqCount();
+ if (kept > 0) {
+ lostAtDlq.put(id, kept);
+ lost += kept;
+ }
Expectation e = r.expectation;
if (e != null) {
Expectation.Reading reading = e.read(now);
@@ -168,7 +178,11 @@ final class ChaosExperiment {
jo.put("ruleTexts", texts);
jo.put("faults", faults);
jo.put("expectations", expectations);
- jo.put("verdict", unmet ? FAIL : !undecided ? PASS : ended ?
INCONCLUSIVE : PENDING);
+ if (lost > 0) {
+ jo.put("lostAtDeadLetterChannel", lost);
+ jo.put("lostAtDeadLetterChannelPerRule", lostAtDlq);
+ }
+ jo.put("verdict", unmet || lost > 0 ? FAIL : !undecided ? PASS : ended
? INCONCLUSIVE : PENDING);
JsonArray routes = new JsonArray();
counts.forEach((route, c) -> {
long[] now0 = read(c);
@@ -225,6 +239,11 @@ final class ChaosExperiment {
sb.append("\n expect
").append(x.getString("expect")).append(": ").append(x.getString("state"))
.append(", ").append(x.getString("value"));
}
+ long lost = jo.getLongOrDefault("lostAtDeadLetterChannel", 0);
+ if (lost > 0) {
+ sb.append("\n lost at the dead letter channel:
").append(lost)
+ .append(lost == 1 ? " message" : " messages").append(",
handled there without being delivered");
+ }
StringJoiner sj = new StringJoiner(", ");
for (Object o : jo.getCollection("routes")) {
JsonObject r = (JsonObject) o;
diff --git
a/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosExperimentTest.java
b/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosExperimentTest.java
index 7b58a415864f..aa4ff153c363 100644
---
a/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosExperimentTest.java
+++
b/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosExperimentTest.java
@@ -52,6 +52,11 @@ class ChaosExperimentTest extends ChaosTestSupport {
@Override
public void configure() {
from("direct:orders").routeId("orders").to("mock:shipped");
+ // a dead letter channel that parks what could not be charged
+ from("direct:billing").routeId("billing")
+ .errorHandler(deadLetterChannel("direct:parked"))
+ .to("mock:charged");
+ from("direct:parked").routeId("parked").to("mock:parked");
}
};
}
@@ -135,6 +140,27 @@ class ChaosExperimentTest extends ChaosTestSupport {
.containsExactly("r1", "r2");
}
+ @Test
+ void messagesLostAtTheDeadLetterChannelFailTheExperiment() {
+ chaos().startExploration("10m", null);
+ // the service is down and the dead letter channel fails too: the dead
letter channel handles each message it
+ // could not deliver, so billing does not fail, and only the count of
the rule sees the loss (CAMEL-26789)
+ chaos().beginExperiment("nothing is lost when billing and parking
fail",
+ "endpoint=mock:charged error expect=billing:failed<=0
within=1m; endpoint=direct:parked error at=dlq");
+ for (int i = 0; i < 3; i++) {
+ template.send("direct:billing", e ->
e.getMessage().setBody("bill"));
+ }
+
+ Map<String, Object> ended = chaos().endExperiment(null);
+
+ assertThat(ended).containsEntry("verdict",
"FAIL").containsEntry("lostAtDeadLetterChannel", 3L);
+ assertThat(((JsonObject)
ended.get("lostAtDeadLetterChannelPerRule")).getLong("r2")).isEqualTo(3);
+ // the expectation alone sees nothing wrong: nothing failed in billing
+ assertThat(first(new JsonObject(ended),
"expectations").getLong("actual")).isZero();
+ assertThat(chaos().report().toText()).contains("1 FAIL nothing is
lost when billing and parking fail",
+ "lost at the dead letter channel: 3 messages, handled there
without being delivered");
+ }
+
@Test
void oneAtATimeWithAnExpectationAndInAnExploration() {
String rule = "endpoint=mock:shipped error expect=orders:failed<=0
within=1m";
diff --git
a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/process/ListChaos.java
b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/process/ListChaos.java
index ed5f4bc11332..4d26bf1f35c1 100644
---
a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/process/ListChaos.java
+++
b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/process/ListChaos.java
@@ -389,6 +389,11 @@ public class ListChaos extends ProcessWatchCommand {
+ x.getString("value"));
}
}
+ long lost = experiment.getLongOrDefault("lostAtDeadLetterChannel", 0);
+ if (lost > 0) {
+ lines.add(more + "lost at the dead letter channel: " + lost +
(lost == 1 ? " message" : " messages")
+ + ", handled there without being delivered");
+ }
List<Object> routes = experiment.getCollection("routes");
if (routes != null && !routes.isEmpty()) {
lines.add(more + "routes: " + routeCounts(routes));
diff --git
a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/process/ListChaosTest.java
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/process/ListChaosTest.java
index 6763a6dfd9f9..6c06f3e46c9c 100644
---
a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/process/ListChaosTest.java
+++
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/process/ListChaosTest.java
@@ -220,6 +220,7 @@ class ListChaosTest extends ProcessCommandTestSupport {
during.put("failed", 9L);
experiment.put("routes", new JsonArray(List.of(during)));
experiment.put("note", "orders has no error handler");
+ experiment.put("lostAtDeadLetterChannel", 58L);
exploration.put("experiments", new JsonArray(List.of(experiment)));
((JsonObject) chaos.get("report")).put("exploration", exploration);
writeStatus(chaos);
@@ -230,7 +231,9 @@ class ListChaosTest extends ProcessCommandTestSupport {
assertThat(out).contains("exploration: running, 6m12s left of 10m0s,
focus: orders",
"rules tried: r1, r2", "baseline (before the first rule):
orders 30 completed, 0 failed",
"experiments:", "1 FAIL an order never fails (r1, r2, ended
after 1m4s)",
- "expect orders:failed<=0: unmet, 9 failed", "routes: orders 41
completed, 9 failed",
+ "expect orders:failed<=0: unmet, 9 failed",
+ "lost at the dead letter channel: 58 messages, handled there
without being delivered",
+ "routes: orders 41 completed, 9 failed",
"note: orders has no error handler");
}
