This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch CAMEL-26675-chaos-report-accuracy in repository https://gitbox.apache.org/repos/asf/camel.git
commit 7674c11f15c41731f8d92160c29afe809b7a9eee Author: Claus Ibsen <[email protected]> AuthorDate: Fri Nov 6 09:36:36 2026 +0100 CAMEL-26675: camel-chaos - report accuracy - The send of the dead letter channel is a hook of its own (at=dlq): an endpoint= or component= rule no longer fails it by accident, and a rule with at=dlq that keeps a message from the dead letter channel counts it as lost, with a FAIL verdict. - A route counts an exchange once (the last pass), not once per pass: direct routes called in a loop or again by a redelivery, expectations and the entry faults. - An expectation of a removed or changed rule, or of a stopped context, is cancelled, so the report can be final again. - chaos-clear --reset (also reset=true of the MCP and camel ask tool, and ChaosManager.reset()) starts a new report. - The rules of camel.chaos.rules start their clock when the context has started. - The report lists the rules by id, and the versions of an id from the latest. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../org/apache/camel/catalog/docs/chaos.adoc | 75 ++++++++----- components/camel-chaos/src/main/docs/chaos.adoc | 75 ++++++++----- .../camel/component/chaos/ChaosEventNotifier.java | 36 +++++- .../camel/component/chaos/ChaosExchangeState.java | 107 ++++++++++++++++++ .../apache/camel/component/chaos/ChaosFaults.java | 21 ++-- .../apache/camel/component/chaos/ChaosManager.java | 123 +++++++++++++++++---- .../apache/camel/component/chaos/ChaosReport.java | 109 ++++++++++++------ .../component/chaos/ChaosRoutePolicyFactory.java | 5 +- .../apache/camel/component/chaos/ChaosRule.java | 60 ++++++++-- .../camel/component/chaos/ChaosRuleParser.java | 10 +- .../camel/component/chaos/ChaosSendHook.java | 19 +++- .../camel/component/chaos/ChaosReportTest.java | 54 ++++++--- .../component/chaos/ChaosRouteCountsTest.java | 62 +++++++++++ .../camel/component/chaos/ChaosRuleParserTest.java | 11 ++ .../camel/component/chaos/ChaosRuleTest.java | 8 +- .../component/chaos/ChaosStartupClockTest.java | 107 ++++++++++++++++++ .../chaos/ChaosStepAndErrorHandlerTest.java | 64 ++++++++++- .../modules/ROOT/pages/camel-jbang-mcp.adoc | 3 +- .../camel-jbang-cmd-chaos-clear.adoc | 1 + .../apache/camel/cli/connector/ChaosAction.java | 7 +- .../LocalCliConnectorChaosActionTest.java | 17 ++- .../META-INF/camel-jbang-commands-metadata.json | 2 +- .../commands/action/CamelChaosClearAction.java | 16 ++- .../dsl/jbang/core/commands/ai/ToolRegistry.java | 16 ++- .../dsl/jbang/core/commands/process/ListChaos.java | 5 +- .../jbang/core/commands/process/ListChaosTest.java | 9 +- .../dsl/jbang/core/commands/mcp/RuntimeTools.java | 11 +- .../core/commands/mcp/RuntimeToolsChaosTest.java | 15 ++- 28 files changed, 864 insertions(+), 184 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/chaos.adoc b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/chaos.adoc index 37621b180b9a..3c2360541ef8 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/chaos.adoc +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/chaos.adoc @@ -68,7 +68,8 @@ report shows it. |`camel.chaos.rules` | |Rules to add at startup, separated by `;`. A rule that is not valid stops the application from starting. They are read once, when the context starts: changing them while the application runs (also in `camel run --dev`) does nothing until the application starts again; add, change and remove rules while it runs with -`ChaosManager`. +`ChaosManager`. Their clock starts when the context has started, so their `until`, `within` and lifecycle times do not +include the startup. |`camel.chaos.maxDuration` |`10m` |The longest time a rule runs: a rule without an `until` duration ends after it, and a longer `until` duration, `delay`, or `duration` of `stall` and `lifecycle`, is rejected when the rule is added. |`camel.chaos.reportOnShutdown` |`true` |Log the report when the context stops, if any rule was added. @@ -107,14 +108,17 @@ camel cmd chaos-add myapp --rule='endpoint=kafka:orders* delay duration=2s rate= camel cmd chaos-change myapp --id=r1 --rule='endpoint=kafka:orders* delay duration=5s rate=0.5 until=5m' camel cmd chaos-remove myapp --id=r1 camel cmd chaos-clear +camel cmd chaos-clear myapp --reset camel get chaos camel get chaos myapp --report ---- `chaos-add` prints the id of the new rule (`r1`, `r2`, ...), and `chaos-change` the version of the rule that replaces it. The ids are per integration, so `chaos-add`, `chaos-change` and `chaos-remove` need the name or pid of one -integration; `chaos-clear` without one acts on every running integration that has chaos switched on. An id that does -not exist is "not found" (exit code 3). `camel get chaos` lists the current rules with their id, version, state, +integration; `chaos-clear` without one acts on every running integration that has chaos switched on. The rules that +`chaos-clear` removes stay in the report; with `--reset` the report starts again, for a new experiment: no rules, +route counts or verdict, as for an application that just started (with the same seed, and the ids of new rules going on +from the last one). An id that does not exist is "not found" (exit code 3). `camel get chaos` lists the current rules with their id, version, state, matches and faults, and `--report` shows the report (also as JSON with `--json`). The commands go through the `chaos` action of the CLI connector, which calls `ChaosManager`: a rule goes through the @@ -132,7 +136,8 @@ as tools, for an AI agent that tries a route against failures: `camel_runtime_ch `camel_runtime_chaos_add`, `camel_runtime_chaos_change`, `camel_runtime_chaos_remove` and `camel_runtime_chaos_clear`. They take the same one-line rules: adding, changing, removing and clearing go through the same `chaos` action of the CLI connector, and the list and the report read the console from the status of the application. So chaos must be switched -on, and a rule that is not valid is refused with the reason. Adding and changing a rule are marked destructive, so a +on, and a rule that is not valid is refused with the reason; `camel_runtime_chaos_clear` takes `reset=true` as +`chaos-clear --reset` does. Adding and changing a rule are marked destructive, so a client that honours `destructiveHint` asks before it runs them, and the `read-write` access level of the MCP security layer does not offer them; the list and the report are read-only. `camel ask` has the same tools, and asks before it adds or changes a rule. The development MCP tools that `camel run --mcp` (`camel.jbang.mcp=true`) serves over HTTP have @@ -186,8 +191,9 @@ target offers, is rejected when it is added, with an error that names the hooks |Target |Hooks (`at=`) |Faults |`endpoint=`, `component=` -|The sends to a matching endpoint (`send`), and the polls of its consumer (`poll`). -|`send`: `error`, `delay`, `drop`, `corrupt`. `poll`: `error`, `delay`, `drop`. +|The sends to a matching endpoint (`send`), the send of the dead letter channel to it (`dlq`, only with `at=dlq`), and +the polls of its consumer (`poll`). +|`send` and `dlq`: `error`, `delay`, `drop`, `corrupt`. `poll`: `error`, `delay`, `drop`. |`route=` |The messages that enter the route (`entry`), and the route itself. @@ -285,7 +291,15 @@ or adds it back after the duration. See <<_route_lifecycle_faults,Route lifecycl `routingSlip`, `dynamicRouter`, a `ProducerTemplate`, a deferred producer), through the send hook of camel-core, inside the error handler that handles a failure of the producer. So an `error` is a real failure for `onException`, redelivery and the dead letter channel, and each redelivery asks the rules again. A send from a `ProducerTemplate` has no error -handler: the exception is returned to the caller. +handler: the exception is returned to the caller. The send of the dead letter channel is not a `send` here, see `dlq`. + +|`dlq` +|The send of the dead letter channel (`deadLetterChannel(...)`) to its endpoint, while it delivers a failed exchange. +Only a rule with `at=dlq` reaches it, so `endpoint=*` or a `component=` rule for the component of the dead letter +channel does not lose the messages that the dead letter channel keeps. When a rule fails or drops that send, the +message does not reach the dead letter channel; when the dead letter channel then handles the exchange (with +`deadLetterHandleNewException`, the default, for a failure), the message is gone, and the report counts it as lost +(see <<_the_report,The report>>). A `delay` or `corrupt` there delivers the message, late or changed. |`poll` |The polls of the scheduled poll consumers of the routes, around their poll strategy (also one set with `pollStrategy`). @@ -309,10 +323,9 @@ that have no error handler of their own, such as inside `doTry` or with camel-jt processors of the step, and a `delay` waits before them. A step that is disabled is not asked. |=== -The send to the dead letter channel is a send too, and the steps inside `onException` and `doCatch` are steps of the -route: so `endpoint=*`, or a `component=` rule for the component of the dead letter channel, also fails the send to the -dead letter channel, and `step=*` also the steps that handle a failure. Name the endpoints or steps under test in the -pattern to keep them out. +The steps inside `onException` and `doCatch` are steps of the route, and the sends there (other than the one of the dead +letter channel) are sends: so `step=*` also fails the steps that handle a failure, and `endpoint=*` the sends of +`onException`. Name the endpoints or steps under test in the pattern to keep them out. == Route lifecycle faults @@ -384,14 +397,18 @@ removed while the route was down, and the `reason` when it did not go as planned `ChaosManager.report()` gives the report, as plain text (`toText()`) for a person and as JSON (`toJson()`) for a tool, and it is logged when the context stops (`camel.chaos.reportOnShutdown`), if any rule was added. It states the seed, each rule with its id and version, its matches, the faults that fired at each hook and the ones that were skipped, its -state (`active`, `ended`, `removed` or `replaced`), and per route: - -* `completed` and `failed`: the exchanges that went through the route, each counted once when the route is done with -it. An exchange counts in every route it goes through: a route called with `direct` counts it too, the copy of a wire -tap or of a seda queue counts in the route that consumes it, and the copies of a split or multicast count in the routes -they are sent to. A message that fails and is read again by its consumer is one failed and one completed exchange. An -exchange marked rollback only counts as failed (kept by the consumer only for the JMS consumers below, with -`markRollbackOnly`). +state (`active`, `ended`, `removed` or `replaced`), and per route. The rules are in the order of their ids, and the +versions of an id from the latest. Per route: + +* `completed` and `failed`: the exchanges that went through the route, each counted once, with how the route was done +with it the last time. An exchange counts in every route it goes through: a route called with `direct` counts it too, +the copy of a wire tap or of a seda queue counts in the route that consumes it, and the copies of a split or multicast +count in the routes they are sent to. An exchange that goes through a route more than once (a `direct` route called in +a loop, or called again by a redelivery of the calling route) is still one exchange of that route: a failed pass and +then a completed one count as one completed exchange. A message that fails and is read again by its consumer is +another exchange: one failed and one completed exchange. An exchange marked rollback only counts as failed (kept by +the consumer only for the JMS consumers below, with `markRollbackOnly`). The `entry` faults of a route are asked once +per exchange too. * `failedKeptByConsumer`: the failed exchanges whose message the consumer of the route keeps and reads again: `file`, `ftp`, `sftp`, `ftps` without `moveFailed`; `jms`, `activemq`, `activemq6`, `amqp` with `transacted=true`, or with `acknowledgementModeName=CLIENT_ACKNOWLEDGE` (the endpoint option; the number set with `setAcknowledgementMode` in Java @@ -406,17 +423,21 @@ calling routes too. The routes are counted while there are rules (also ended ones). Camel does not call the route policies while the context stops, so the exchanges that finish while the context drains them are not counted. -Without an expectation the verdict is PASS, as chaos makes exchanges fail on purpose, and the report never says that a -message was lost. An expectation, `expect=orders:100 within=2m`, says that route `orders` completes 100 more exchanges +Without an expectation the verdict is PASS, as chaos makes exchanges fail on purpose, and the report does not say that a +message was lost, except for the messages that a rule with `at=dlq` kept from reaching the dead letter channel, which +then handled them: those count as lost (in the report of the rule as `lostAtDlq`), and make the verdict FAIL, with the +reason. An expectation, `expect=orders:100 within=2m`, says that route `orders` completes 100 more exchanges after the rule is added and before the time is up (by default the `until` time of the rule, else -`camel.chaos.maxDuration`, which is also the longest `within`). An exchange that the error handler handled (for example -one that went to the dead letter channel) completes, and so counts as completed, not as lost. The route id is not +`camel.chaos.maxDuration`, which is also the longest `within`). An exchange of the route counts once, the first time +it completes in the route. An exchange that the error handler handled (for example one that went to the dead letter +channel) completes, and so counts as completed, not as lost. The route id is not checked when the rule is added, as the route may come later (a WARN says so when a rule is added or changed, or the context starts, and no route has that id). An expectation is met when the count is reached in time, and unmet when the time is up without it: then the verdict is FAIL, with the reason, and the missing exchanges count as lost (100 expected, 97 completed: 3 lost). Before that it is pending, and so is the report (`isFinal()` is false): a tool must treat a PASS -with a pending expectation as not decided yet. An expectation stays pending for good when its rule is removed or changed -first, or when the context stops first (also when the context is started again), as the messages may still arrive. +with a pending expectation as not decided yet. An expectation is cancelled when its rule is removed or changed first, or +when the context stops first (also when the context is started again): the messages may still arrive, so it is +neither met nor unmet, and it does not keep the report from being final. == Time and the seed @@ -445,8 +466,8 @@ of the rules (an add, change or remove) leaves it out of what the hooks read, as decided. * At most 100 rules at a time. When there are 100, adding a rule first moves the rules that ended (and whose expectation is decided) into the report history, where their ids can no longer be changed or removed; the report -keeps the last 100 removed, replaced or ended ones. A pending expectation of a rule that leaves the history keeps the -report not final. An unmet expectation of a rule that leaves the report still counts in the verdict and in `lost`. +keeps the last 100 removed, replaced or ended ones. An unmet expectation of a rule that leaves the report still counts +in the verdict and in `lost`. * A rule is at most 2000 characters long. * There is one send hook and one step hook per `CamelContext`: when another one is installed already, chaos keeps it, logs a WARN, and its rules do not fire at those hooks. diff --git a/components/camel-chaos/src/main/docs/chaos.adoc b/components/camel-chaos/src/main/docs/chaos.adoc index 37621b180b9a..3c2360541ef8 100644 --- a/components/camel-chaos/src/main/docs/chaos.adoc +++ b/components/camel-chaos/src/main/docs/chaos.adoc @@ -68,7 +68,8 @@ report shows it. |`camel.chaos.rules` | |Rules to add at startup, separated by `;`. A rule that is not valid stops the application from starting. They are read once, when the context starts: changing them while the application runs (also in `camel run --dev`) does nothing until the application starts again; add, change and remove rules while it runs with -`ChaosManager`. +`ChaosManager`. Their clock starts when the context has started, so their `until`, `within` and lifecycle times do not +include the startup. |`camel.chaos.maxDuration` |`10m` |The longest time a rule runs: a rule without an `until` duration ends after it, and a longer `until` duration, `delay`, or `duration` of `stall` and `lifecycle`, is rejected when the rule is added. |`camel.chaos.reportOnShutdown` |`true` |Log the report when the context stops, if any rule was added. @@ -107,14 +108,17 @@ camel cmd chaos-add myapp --rule='endpoint=kafka:orders* delay duration=2s rate= camel cmd chaos-change myapp --id=r1 --rule='endpoint=kafka:orders* delay duration=5s rate=0.5 until=5m' camel cmd chaos-remove myapp --id=r1 camel cmd chaos-clear +camel cmd chaos-clear myapp --reset camel get chaos camel get chaos myapp --report ---- `chaos-add` prints the id of the new rule (`r1`, `r2`, ...), and `chaos-change` the version of the rule that replaces it. The ids are per integration, so `chaos-add`, `chaos-change` and `chaos-remove` need the name or pid of one -integration; `chaos-clear` without one acts on every running integration that has chaos switched on. An id that does -not exist is "not found" (exit code 3). `camel get chaos` lists the current rules with their id, version, state, +integration; `chaos-clear` without one acts on every running integration that has chaos switched on. The rules that +`chaos-clear` removes stay in the report; with `--reset` the report starts again, for a new experiment: no rules, +route counts or verdict, as for an application that just started (with the same seed, and the ids of new rules going on +from the last one). An id that does not exist is "not found" (exit code 3). `camel get chaos` lists the current rules with their id, version, state, matches and faults, and `--report` shows the report (also as JSON with `--json`). The commands go through the `chaos` action of the CLI connector, which calls `ChaosManager`: a rule goes through the @@ -132,7 +136,8 @@ as tools, for an AI agent that tries a route against failures: `camel_runtime_ch `camel_runtime_chaos_add`, `camel_runtime_chaos_change`, `camel_runtime_chaos_remove` and `camel_runtime_chaos_clear`. They take the same one-line rules: adding, changing, removing and clearing go through the same `chaos` action of the CLI connector, and the list and the report read the console from the status of the application. So chaos must be switched -on, and a rule that is not valid is refused with the reason. Adding and changing a rule are marked destructive, so a +on, and a rule that is not valid is refused with the reason; `camel_runtime_chaos_clear` takes `reset=true` as +`chaos-clear --reset` does. Adding and changing a rule are marked destructive, so a client that honours `destructiveHint` asks before it runs them, and the `read-write` access level of the MCP security layer does not offer them; the list and the report are read-only. `camel ask` has the same tools, and asks before it adds or changes a rule. The development MCP tools that `camel run --mcp` (`camel.jbang.mcp=true`) serves over HTTP have @@ -186,8 +191,9 @@ target offers, is rejected when it is added, with an error that names the hooks |Target |Hooks (`at=`) |Faults |`endpoint=`, `component=` -|The sends to a matching endpoint (`send`), and the polls of its consumer (`poll`). -|`send`: `error`, `delay`, `drop`, `corrupt`. `poll`: `error`, `delay`, `drop`. +|The sends to a matching endpoint (`send`), the send of the dead letter channel to it (`dlq`, only with `at=dlq`), and +the polls of its consumer (`poll`). +|`send` and `dlq`: `error`, `delay`, `drop`, `corrupt`. `poll`: `error`, `delay`, `drop`. |`route=` |The messages that enter the route (`entry`), and the route itself. @@ -285,7 +291,15 @@ or adds it back after the duration. See <<_route_lifecycle_faults,Route lifecycl `routingSlip`, `dynamicRouter`, a `ProducerTemplate`, a deferred producer), through the send hook of camel-core, inside the error handler that handles a failure of the producer. So an `error` is a real failure for `onException`, redelivery and the dead letter channel, and each redelivery asks the rules again. A send from a `ProducerTemplate` has no error -handler: the exception is returned to the caller. +handler: the exception is returned to the caller. The send of the dead letter channel is not a `send` here, see `dlq`. + +|`dlq` +|The send of the dead letter channel (`deadLetterChannel(...)`) to its endpoint, while it delivers a failed exchange. +Only a rule with `at=dlq` reaches it, so `endpoint=*` or a `component=` rule for the component of the dead letter +channel does not lose the messages that the dead letter channel keeps. When a rule fails or drops that send, the +message does not reach the dead letter channel; when the dead letter channel then handles the exchange (with +`deadLetterHandleNewException`, the default, for a failure), the message is gone, and the report counts it as lost +(see <<_the_report,The report>>). A `delay` or `corrupt` there delivers the message, late or changed. |`poll` |The polls of the scheduled poll consumers of the routes, around their poll strategy (also one set with `pollStrategy`). @@ -309,10 +323,9 @@ that have no error handler of their own, such as inside `doTry` or with camel-jt processors of the step, and a `delay` waits before them. A step that is disabled is not asked. |=== -The send to the dead letter channel is a send too, and the steps inside `onException` and `doCatch` are steps of the -route: so `endpoint=*`, or a `component=` rule for the component of the dead letter channel, also fails the send to the -dead letter channel, and `step=*` also the steps that handle a failure. Name the endpoints or steps under test in the -pattern to keep them out. +The steps inside `onException` and `doCatch` are steps of the route, and the sends there (other than the one of the dead +letter channel) are sends: so `step=*` also fails the steps that handle a failure, and `endpoint=*` the sends of +`onException`. Name the endpoints or steps under test in the pattern to keep them out. == Route lifecycle faults @@ -384,14 +397,18 @@ removed while the route was down, and the `reason` when it did not go as planned `ChaosManager.report()` gives the report, as plain text (`toText()`) for a person and as JSON (`toJson()`) for a tool, and it is logged when the context stops (`camel.chaos.reportOnShutdown`), if any rule was added. It states the seed, each rule with its id and version, its matches, the faults that fired at each hook and the ones that were skipped, its -state (`active`, `ended`, `removed` or `replaced`), and per route: - -* `completed` and `failed`: the exchanges that went through the route, each counted once when the route is done with -it. An exchange counts in every route it goes through: a route called with `direct` counts it too, the copy of a wire -tap or of a seda queue counts in the route that consumes it, and the copies of a split or multicast count in the routes -they are sent to. A message that fails and is read again by its consumer is one failed and one completed exchange. An -exchange marked rollback only counts as failed (kept by the consumer only for the JMS consumers below, with -`markRollbackOnly`). +state (`active`, `ended`, `removed` or `replaced`), and per route. The rules are in the order of their ids, and the +versions of an id from the latest. Per route: + +* `completed` and `failed`: the exchanges that went through the route, each counted once, with how the route was done +with it the last time. An exchange counts in every route it goes through: a route called with `direct` counts it too, +the copy of a wire tap or of a seda queue counts in the route that consumes it, and the copies of a split or multicast +count in the routes they are sent to. An exchange that goes through a route more than once (a `direct` route called in +a loop, or called again by a redelivery of the calling route) is still one exchange of that route: a failed pass and +then a completed one count as one completed exchange. A message that fails and is read again by its consumer is +another exchange: one failed and one completed exchange. An exchange marked rollback only counts as failed (kept by +the consumer only for the JMS consumers below, with `markRollbackOnly`). The `entry` faults of a route are asked once +per exchange too. * `failedKeptByConsumer`: the failed exchanges whose message the consumer of the route keeps and reads again: `file`, `ftp`, `sftp`, `ftps` without `moveFailed`; `jms`, `activemq`, `activemq6`, `amqp` with `transacted=true`, or with `acknowledgementModeName=CLIENT_ACKNOWLEDGE` (the endpoint option; the number set with `setAcknowledgementMode` in Java @@ -406,17 +423,21 @@ calling routes too. The routes are counted while there are rules (also ended ones). Camel does not call the route policies while the context stops, so the exchanges that finish while the context drains them are not counted. -Without an expectation the verdict is PASS, as chaos makes exchanges fail on purpose, and the report never says that a -message was lost. An expectation, `expect=orders:100 within=2m`, says that route `orders` completes 100 more exchanges +Without an expectation the verdict is PASS, as chaos makes exchanges fail on purpose, and the report does not say that a +message was lost, except for the messages that a rule with `at=dlq` kept from reaching the dead letter channel, which +then handled them: those count as lost (in the report of the rule as `lostAtDlq`), and make the verdict FAIL, with the +reason. An expectation, `expect=orders:100 within=2m`, says that route `orders` completes 100 more exchanges after the rule is added and before the time is up (by default the `until` time of the rule, else -`camel.chaos.maxDuration`, which is also the longest `within`). An exchange that the error handler handled (for example -one that went to the dead letter channel) completes, and so counts as completed, not as lost. The route id is not +`camel.chaos.maxDuration`, which is also the longest `within`). An exchange of the route counts once, the first time +it completes in the route. An exchange that the error handler handled (for example one that went to the dead letter +channel) completes, and so counts as completed, not as lost. The route id is not checked when the rule is added, as the route may come later (a WARN says so when a rule is added or changed, or the context starts, and no route has that id). An expectation is met when the count is reached in time, and unmet when the time is up without it: then the verdict is FAIL, with the reason, and the missing exchanges count as lost (100 expected, 97 completed: 3 lost). Before that it is pending, and so is the report (`isFinal()` is false): a tool must treat a PASS -with a pending expectation as not decided yet. An expectation stays pending for good when its rule is removed or changed -first, or when the context stops first (also when the context is started again), as the messages may still arrive. +with a pending expectation as not decided yet. An expectation is cancelled when its rule is removed or changed first, or +when the context stops first (also when the context is started again): the messages may still arrive, so it is +neither met nor unmet, and it does not keep the report from being final. == Time and the seed @@ -445,8 +466,8 @@ of the rules (an add, change or remove) leaves it out of what the hooks read, as decided. * At most 100 rules at a time. When there are 100, adding a rule first moves the rules that ended (and whose expectation is decided) into the report history, where their ids can no longer be changed or removed; the report -keeps the last 100 removed, replaced or ended ones. A pending expectation of a rule that leaves the history keeps the -report not final. An unmet expectation of a rule that leaves the report still counts in the verdict and in `lost`. +keeps the last 100 removed, replaced or ended ones. An unmet expectation of a rule that leaves the report still counts +in the verdict and in `lost`. * A rule is at most 2000 characters long. * There is one send hook and one step hook per `CamelContext`: when another one is installed already, chaos keeps it, logs a WARN, and its rules do not fire at those hooks. diff --git a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosEventNotifier.java b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosEventNotifier.java index db17795208ad..e96be10d638d 100644 --- a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosEventNotifier.java +++ b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosEventNotifier.java @@ -17,6 +17,7 @@ package org.apache.camel.component.chaos; import org.apache.camel.Exchange; +import org.apache.camel.ExchangeExtension; import org.apache.camel.Route; import org.apache.camel.spi.CamelEvent; import org.apache.camel.spi.UnitOfWork; @@ -26,6 +27,11 @@ import org.apache.camel.support.EventNotifierSupport; * Counts per route, for the report, the redelivery attempts of the error handlers. A redelivery counts in the route * that runs the step being redelivered. It counts only while there are rules, and asks for no exchange events * otherwise. (The completed and failed exchanges are counted by the chaos route policy of each route.) + * <p/> + * It also marks an exchange while the dead letter channel delivers it, so the send hook tells the send of the dead + * letter channel apart (rules reach it only with {@code at=dlq}), and counts a message as lost when such a rule failed + * or dropped that send and the dead letter channel then handled the exchange (as with + * {@code deadLetterHandleNewException}, the default). */ final class ChaosEventNotifier extends EventNotifierSupport { @@ -37,6 +43,18 @@ final class ChaosEventNotifier extends EventNotifierSupport { @Override public void notify(CamelEvent event) throws Exception { + if (event instanceof CamelEvent.ExchangeFailureHandlingEvent handling) { + if (handling.isDeadLetterChannel()) { + ChaosExchangeState.of(handling.getExchange()).deadLetter = true; + } + return; + } + if (event instanceof CamelEvent.ExchangeFailureHandledEvent handled) { + if (handled.isDeadLetterChannel()) { + deadLetterDone(handled.getExchange()); + } + return; + } if (event instanceof CamelEvent.ExchangeRedeliveryEvent redelivery && manager.isCounting()) { Exchange exchange = redelivery.getExchange(); UnitOfWork uow = exchange.getUnitOfWork(); @@ -48,6 +66,21 @@ final class ChaosEventNotifier extends EventNotifierSupport { } } + private static void deadLetterDone(Exchange exchange) { + ChaosExchangeState state = ChaosExchangeState.peek(exchange); + if (state == null) { + return; + } + state.deadLetter = false; + ChaosRule fault = state.deadLetterFault; + state.deadLetterFault = null; + ExchangeExtension ee = exchange.getExchangeExtension(); + if (fault != null && ee.isErrorHandlerHandledSet() && ee.isErrorHandlerHandled()) { + // the dead letter channel handled the exchange without delivering it + fault.lostAtDlq(); + } + } + @Override public boolean isIgnoreExchangeEvents() { return !manager.isCounting(); @@ -80,7 +113,8 @@ final class ChaosEventNotifier extends EventNotifierSupport { @Override public boolean isIgnoreExchangeFailedEvents() { - return true; + // the failure handling events of the dead letter channel are exchange failed events + return false; } @Override diff --git a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosExchangeState.java b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosExchangeState.java new file mode 100644 index 000000000000..c0ad44e41f23 --- /dev/null +++ b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosExchangeState.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.chaos; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.camel.Exchange; +import org.apache.camel.ExchangeExtension; +import org.apache.camel.SafeCopyProperty; + +/** + * What chaos keeps of one exchange: how each route it went through is done with it, so a route counts the exchange once + * however often it enters the route (a {@code direct} route called in a loop, or again by a redelivery), and whether + * the dead letter channel is delivering it, so the send of the dead letter channel is told apart from the other sends. + * <p/> + * It is a {@link SafeCopyProperty} of the exchange: internal to Camel, not one of the properties of the exchange. A + * copy of the exchange (a wire tap, a split, a seda queue) is another exchange, and starts with a state of its own. An + * exchange is processed by one thread at a time, so the state needs no lock. + */ +final class ChaosExchangeState implements SafeCopyProperty { + + static final String KEY = "CamelChaosExchangeState"; + + /** How a route was done with the exchange the last time, for its counts. */ + enum Outcome { + // entered the route (the entry faults were asked), not done with yet + ENTERED, + COMPLETED, + // completed after the error handler handled a failure + HANDLED, + FAILED, + // failed, and the consumer of the route reads the message again + FAILED_KEPT; + + boolean completed() { + return this == COMPLETED || this == HANDLED; + } + } + + /** A route the exchange went through. */ + static final class RouteState { + Outcome outcome = Outcome.ENTERED; + // the counts of the route the outcome is counted in: other counts (after a reset of the report) have not + // counted the exchange + ChaosManager.RouteCounts counts; + // the expectations of the route counted the exchange as completed (once, also when it completes again) + boolean expected; + } + + private final Map<String, RouteState> routes = new HashMap<>(4); + // the dead letter channel is delivering the exchange + boolean deadLetter; + // the rule that failed or dropped that delivery, which loses the message when the dead letter channel then handles + // it + ChaosRule deadLetterFault; + + /** The state of the exchange, created when there is none. */ + static ChaosExchangeState of(Exchange exchange) { + ExchangeExtension ee = exchange.getExchangeExtension(); + ChaosExchangeState state = ee.getSafeCopyProperty(KEY, ChaosExchangeState.class); + if (state == null) { + state = new ChaosExchangeState(); + ee.setSafeCopyProperty(KEY, state); + } + return state; + } + + /** The state of the exchange, or null when chaos has not seen it. */ + static ChaosExchangeState peek(Exchange exchange) { + return exchange.getExchangeExtension().getSafeCopyProperty(KEY, ChaosExchangeState.class); + } + + /** + * The exchange enters the route. + * + * @return whether it is the first time, so the entry faults are asked once per exchange + */ + boolean enter(String routeId) { + return routes.putIfAbsent(routeId, new RouteState()) == null; + } + + /** The state of the exchange in the route, created when the exchange enters it. */ + RouteState route(String routeId) { + return routes.computeIfAbsent(routeId, k -> new RouteState()); + } + + @Override + public SafeCopyProperty safeCopy() { + // a copy is another exchange: it is counted, and goes through the dead letter channel, on its own + return new ChaosExchangeState(); + } +} diff --git a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosFaults.java b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosFaults.java index c4b29e28bb0f..27fe753d172a 100644 --- a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosFaults.java +++ b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosFaults.java @@ -78,30 +78,31 @@ final class ChaosFaults { } /** - * Raises the fault of a rule that fired at a send. + * Raises the fault of a rule that fired at a send: at the {@code send} hook, or the {@code dlq} hook for the send + * of the dead letter channel. * * @return the value for {@link SendHook#beforeSend(Exchange, Endpoint)} */ - static long atSend(ChaosRule rule, Exchange exchange, Endpoint endpoint) { + static long atSend(ChaosRule rule, Exchange exchange, Endpoint endpoint, Hook hook) { switch (rule.fault) { case ERROR -> { if (rule.status > 0) { - return httpFailure(rule, exchange, endpoint); + return httpFailure(rule, exchange, endpoint, hook); } exchange.setException(newException(rule)); - rule.fired(Hook.SEND); + rule.fired(hook); return SendHook.PROCEED; } case DELAY -> { - rule.fired(Hook.SEND); + rule.fired(hook); return rule.duration; } case DROP -> { - rule.fired(Hook.SEND); + rule.fired(hook); return SendHook.SKIP; } case CORRUPT -> { - corrupt(rule, exchange, Hook.SEND); + corrupt(rule, exchange, hook); return SendHook.PROCEED; } default -> throw new IllegalStateException("Unknown fault " + rule.fault); @@ -112,7 +113,7 @@ final class ChaosFaults { * The failure of an HTTP component for the status of the rule, or the response with that status when the endpoint * has {@code throwExceptionOnFailure=false}, as the producer of the component does. The producer is not called. */ - private static long httpFailure(ChaosRule rule, Exchange exchange, Endpoint endpoint) { + private static long httpFailure(ChaosRule rule, Exchange exchange, Endpoint endpoint, Hook hook) { String scheme = ChaosPatterns.scheme(ChaosPatterns.endpointUri(endpoint)); if (!ChaosRuleParser.HTTP_SCHEMES.contains(scheme)) { // the rule matched an endpoint that is not an HTTP one (with a pattern such as endpoint=*inventory*) @@ -126,7 +127,7 @@ final class ChaosFaults { if (Boolean.FALSE.equals(toBoolean(option(endpoint, "throwExceptionOnFailure"))) || isOkStatus(rule.status, option(endpoint, "okStatusCodeRange"))) { response(message, rule, !undertow); - rule.fired(Hook.SEND); + rule.fired(hook); return SendHook.SKIP; } Throwable failure; @@ -152,7 +153,7 @@ final class ChaosFaults { response(message, rule, true); } exchange.setException(failure); - rule.fired(Hook.SEND); + rule.fired(hook); return SendHook.PROCEED; } diff --git a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosManager.java b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosManager.java index 7e8e18d707ef..9af42102d310 100644 --- a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosManager.java +++ b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosManager.java @@ -79,6 +79,7 @@ public final class ChaosManager { private int evictedUnmet; private long evictedLost; private int evictedPending; + private long evictedLostAtDlq; // the snapshot the hooks read private volatile RuleSet ruleSet = RuleSet.EMPTY; @@ -141,6 +142,8 @@ public final class ChaosManager { } String id = "r" + nextId++; ChaosRule added = new ChaosRule(definition, id, seed, now, maxDuration); + // a rule added before the context started starts its clock when it has started + added.clockStarted = camelContext.isStarted(); rules.add(added); update(); LOG.info("Chaos rule {} added: {}", id, added.text); @@ -172,6 +175,7 @@ public final class ChaosManager { checkRoute(definition); long now = TimeSource.currentTimeMillis(); ChaosRule changed = new ChaosRule(definition, id, rules.get(index).version + 1, seed, now, maxDuration); + changed.clockStarted = camelContext.isStarted(); ChaosRule old = rules.set(index, changed); old.removed(State.REPLACED, now); remember(old); @@ -198,7 +202,7 @@ public final class ChaosManager { } /** - * Removes a rule. It stays in the report, and its expectation, if not decided yet, stays pending. + * Removes a rule. It stays in the report, and its expectation, if not decided yet, is cancelled. * * @param id the id of the rule * @return whether there was a rule with that id @@ -240,6 +244,26 @@ public final class ChaosManager { } } + /** + * Removes all rules and starts a new report: the removed rules, the counts per route and the verdict are gone, as + * for an application that just started (with the same seed). The ids of new rules go on from the last one. + * + * @return the number of rules removed + */ + public int reset() { + synchronized (lock) { + int count = clearRules(); + history.clear(); + evictedUnmet = 0; + evictedLost = 0; + evictedPending = 0; + evictedLostAtDlq = 0; + routes.clear(); + LOG.info("Chaos report reset"); + return count; + } + } + /** * The report: the rules and what they did, the counts per route, the expectations and the verdict. */ @@ -249,16 +273,19 @@ public final class ChaosManager { int unmet; long lost; int pending; + long lostAtDlq; synchronized (lock) { current = new ArrayList<>(rules); removed = new ArrayList<>(history); unmet = evictedUnmet; lost = evictedLost; pending = evictedPending; + lostAtDlq = evictedLostAtDlq; } Map<String, RouteCounts> counts = new TreeMap<>(routes); return new ChaosReport( - seed, seedGiven, TimeSource.currentTimeMillis(), current, removed, counts, unmet, lost, pending); + seed, seedGiven, TimeSource.currentTimeMillis(), current, removed, counts, unmet, lost, pending, + lostAtDlq); } // -- for the hooks, the event notifier and the plugin @@ -285,39 +312,55 @@ public final class ChaosManager { * letter channel or {@code onException} with {@code handled(true)}) or failed (also when it was marked rollback * only). An exchange counts in each route it goes through: a route called with {@code direct} counts it too, and a * copy of a wire tap or a seda queue counts in the route that consumes it. + * <p/> + * An exchange counts once per route, with how the route was done with it the last time: camel-core calls the route + * policy each time the exchange goes through the route (a {@code direct} route called in a loop, or again by a + * redelivery of the calling route), so a later pass moves the count (a failed pass and then a completed one count + * as one completed exchange). The expectations of the route count it the first time it completes. */ void routeDone(Route route, Exchange exchange) { String routeId = route.getRouteId(); RouteCounts counts = counts(routeId); + ChaosExchangeState.Outcome outcome = outcome(route, exchange); + ChaosExchangeState.RouteState state = ChaosExchangeState.of(exchange).route(routeId); + ChaosExchangeState.Outcome previous = state.counts == counts ? state.outcome : ChaosExchangeState.Outcome.ENTERED; + if (previous == outcome) { + return; + } + counts.uncount(previous); + counts.count(outcome); + state.outcome = outcome; + state.counts = counts; + if (outcome.completed() && !state.expected) { + state.expected = true; + for (ChaosRule.Expectation expectation : ruleSet.expectations) { + if (expectation.route.equals(routeId)) { + expectation.complete(); + } + } + } + } + + private static ChaosExchangeState.Outcome outcome(Route route, Exchange exchange) { // an exchange marked rollback only has no exception, but its message is not processed either boolean failed = exchange.isFailed(); if (failed || exchange.isRollbackOnly() || exchange.isRollbackOnlyLast()) { - counts.failed.increment(); // the JMS listener rolls back for isRollbackOnly, not for isRollbackOnlyLast (which a transaction policy // of the route handles) - if (failed + boolean kept = failed ? ChaosConsumers.keepsFailed(route.getEndpoint()) - : exchange.isRollbackOnly() && ChaosConsumers.keepsRollbackOnly(route.getEndpoint())) { - counts.failedKept.increment(); - } - return; - } - for (ChaosRule.Expectation expectation : ruleSet.expectations) { - if (expectation.route.equals(routeId)) { - expectation.complete(); - } + : exchange.isRollbackOnly() && ChaosConsumers.keepsRollbackOnly(route.getEndpoint()); + return kept ? ChaosExchangeState.Outcome.FAILED_KEPT : ChaosExchangeState.Outcome.FAILED; } ExchangeExtension ee = exchange.getExchangeExtension(); - if (ee.isErrorHandlerHandledSet() && ee.isErrorHandlerHandled()) { - counts.handled.increment(); - } - counts.completed.increment(); + return ee.isErrorHandlerHandledSet() && ee.isErrorHandlerHandled() + ? ChaosExchangeState.Outcome.HANDLED : ChaosExchangeState.Outcome.COMPLETED; } /** * The context stops: camel-core does not call the route policies while it stops, so the exchanges that finish while - * the context drains them are not counted. The expectations that are not decided yet stay pending for good, as for - * a removed rule, so they are not called unmet for messages that were not counted. + * the context drains them are not counted. The expectations that are not decided yet are cancelled, as for a + * removed rule, so they are not called unmet for messages that were not counted. */ void contextStopping() { synchronized (lock) { @@ -395,13 +438,21 @@ public final class ChaosManager { } /** - * The context started: warns for the expectations of the current rules (such as those of camel.chaos.rules, added - * before the routes) whose route does not exist. It runs at every start of the context. + * The context started: the rules added before it started (such as those of camel.chaos.rules) start their clock, so + * their until, within and lifecycle times do not include the startup, and it warns for the expectations of the + * current rules whose route does not exist. It runs at every start of the context. * * @return the ids of the rules it warned for */ List<String> contextStarted() { synchronized (lock) { + long now = TimeSource.currentTimeMillis(); + for (ChaosRule rule : rules) { + if (!rule.clockStarted) { + rule.clockStarted = true; + rule.startClock(now); + } + } // the lifecycle rules that wait for the context to start act now lifecycle.contextStarted(); update(); @@ -452,6 +503,8 @@ public final class ChaosManager { history.addLast(rule); while (history.size() > MAX_HISTORY) { ChaosRule evicted = history.removeFirst(); + // the messages it lost at the dead letter channel keep the verdict FAIL too + evictedLostAtDlq += evicted.lostAtDlqCount(); ChaosRule.Expectation expectation = evicted.expectation; // an unmet expectation keeps the verdict FAIL after its rule left the report ChaosRule.Expectation.Reading reading @@ -529,6 +582,7 @@ public final class ChaosManager { static final RuleSet EMPTY = new RuleSet(List.of(), 0); final ChaosRule[] send; + final ChaosRule[] dlq; final ChaosRule[] poll; final ChaosRule[] entry; final ChaosRule[] step; @@ -538,6 +592,7 @@ public final class ChaosManager { RuleSet(List<ChaosRule> rules, long now) { List<ChaosRule> live = rules.stream().filter(r -> r.live(now)).toList(); send = live.stream().filter(r -> r.hooks.contains(Hook.SEND)).toArray(ChaosRule[]::new); + dlq = live.stream().filter(r -> r.hooks.contains(Hook.DLQ)).toArray(ChaosRule[]::new); poll = live.stream().filter(r -> r.hooks.contains(Hook.POLL)).toArray(ChaosRule[]::new); entry = live.stream().filter(r -> r.hooks.contains(Hook.ENTRY)).toArray(ChaosRule[]::new); step = live.stream().filter(r -> r.hooks.contains(Hook.STEP)).toArray(ChaosRule[]::new); @@ -559,5 +614,31 @@ public final class ChaosManager { final LongAdder failedKept = new LongAdder(); final LongAdder handled = new LongAdder(); final LongAdder redeliveries = new LongAdder(); + + void count(ChaosExchangeState.Outcome outcome) { + add(outcome, 1); + } + + void uncount(ChaosExchangeState.Outcome outcome) { + add(outcome, -1); + } + + private void add(ChaosExchangeState.Outcome outcome, int n) { + switch (outcome) { + case COMPLETED -> completed.add(n); + case HANDLED -> { + completed.add(n); + handled.add(n); + } + case FAILED -> failed.add(n); + case FAILED_KEPT -> { + failed.add(n); + failedKept.add(n); + } + default -> { + // entered, not counted + } + } + } } } diff --git a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosReport.java b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosReport.java index fd4b9c239c8e..260065d199b0 100644 --- a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosReport.java +++ b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosReport.java @@ -18,6 +18,7 @@ package org.apache.camel.component.chaos; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Set; @@ -35,14 +36,18 @@ import org.apache.camel.util.json.JsonObject; * What the chaos rules did, as of the time the report was made: the seed, each rule with its matches and the faults * that fired, the counts per route, the expectations and the verdict. * <p/> - * Without an expectation the verdict is PASS, as chaos makes exchanges fail on purpose. Only an expectation that is - * unmet (its route did not complete that many exchanges in time) makes it FAIL, and only an unmet expectation counts - * messages as lost. An expectation that is not decided yet is pending, and so is the report: a PASS with a pending - * expectation is not final ({@link #isFinal()}). + * Without an expectation the verdict is PASS, as chaos makes exchanges fail on purpose. An expectation that is unmet + * (its route did not complete that many exchanges in time) makes it FAIL, and counts messages as lost; so does a rule + * with {@code at=dlq} that failed or dropped the send of the dead letter channel, which then handled the exchange + * without delivering it. An expectation that is not decided yet is pending, and so is the report: a PASS with a pending + * expectation is not final ({@link #isFinal()}). An expectation whose rule was removed or changed, or whose context + * stopped, before it was decided is cancelled: it is neither met nor unmet, and does not keep the report pending. * <p/> - * A route counts each exchange that goes through it once, when the route is done with it: a message that fails and is - * read again by its consumer is one failed and one completed exchange. A failed exchange is "kept by the consumer" when - * the consumer of the route reads the message again (for example a file consumer without {@code moveFailed}). + * A route counts each exchange that goes through it once, with how the route was done with it the last time (an + * exchange can go through a route more than once, such as a {@code direct} route called in a loop): a message that + * fails and is read again by its consumer is one failed and one completed exchange. A failed exchange is "kept by the + * consumer" when the consumer of the route reads the message again (for example a file consumer without + * {@code moveFailed}). * * @since 4.24 */ @@ -55,26 +60,37 @@ public final class ChaosReport { private final long seed; private final boolean seedGiven; + // in the order of their ids, and the versions of an id from the latest private final List<RuleEntry> rules = new ArrayList<>(); - // the current rules come first in rules, then the removed, replaced and retired ones - private final int current; private final List<RouteEntry> routes = new ArrayList<>(); private final List<String> reasons = new ArrayList<>(); private final long lost; private final boolean pending; ChaosReport(long seed, boolean seedGiven, long now, List<ChaosRule> current, List<ChaosRule> removed, - Map<String, RouteCounts> counts, int evictedUnmet, long evictedLost, int evictedPending) { + Map<String, RouteCounts> counts, int evictedUnmet, long evictedLost, int evictedPending, + long evictedLostAtDlq) { this.seed = seed; this.seedGiven = seedGiven; long lostCount = 0; boolean anyPending = false; - this.current = current.size(); - List<ChaosRule> all = new ArrayList<>(current); - all.addAll(removed); - for (ChaosRule rule : all) { - RuleEntry entry = new RuleEntry(rule, now); - rules.add(entry); + for (ChaosRule rule : current) { + rules.add(new RuleEntry(rule, now, true)); + } + for (ChaosRule rule : removed) { + rules.add(new RuleEntry(rule, now, false)); + } + rules.sort(Comparator.comparingInt((RuleEntry r) -> idNumber(r.id)) + .thenComparing(Comparator.comparingInt((RuleEntry r) -> r.version).reversed())); + for (RuleEntry entry : rules) { + if (entry.lostAtDlq > 0) { + lostCount += entry.lostAtDlq; + reasons.add("rule " + entry.id + " failed or dropped " + entry.lostAtDlq + + (entry.lostAtDlq == 1 ? " send" : " sends") + " of the dead letter channel, which handled" + + (entry.lostAtDlq == 1 ? " it without delivering it: " : " them without delivering them: ") + + entry.lostAtDlq + " lost"); + } + ChaosRule rule = entry.rule; if (entry.outcome == Outcome.UNMET) { long missing = Math.max(0, rule.expectation.count - entry.completed); lostCount += missing; @@ -90,13 +106,28 @@ public final class ChaosReport { reasons.add(evictedUnmet + " unmet expectations of rules that are no longer in the report: " + evictedLost + " lost"); } + if (evictedLostAtDlq > 0) { + lostCount += evictedLostAtDlq; + reasons.add("rules that are no longer in the report kept messages from the dead letter channel: " + + evictedLostAtDlq + " lost"); + } counts.forEach((route, c) -> routes.add(new RouteEntry(route, c))); this.lost = lostCount; this.pending = anyPending || evictedPending > 0; } + // the number of an id such as r12, for the order of the rules + private static int idNumber(String id) { + try { + return Integer.parseInt(id.substring(1)); + } catch (RuntimeException e) { + return Integer.MAX_VALUE; + } + } + /** - * The verdict: {@link #FAIL} when an expectation is unmet, {@link #PASS} otherwise. + * The verdict: {@link #FAIL} when an expectation is unmet, or a message was lost at the dead letter channel; + * {@link #PASS} otherwise. */ public String getVerdict() { return reasons.isEmpty() ? PASS : FAIL; @@ -110,15 +141,16 @@ public final class ChaosReport { } /** - * The reasons of a {@link #FAIL} verdict, one per unmet expectation. + * The reasons of a {@link #FAIL} verdict, one per unmet expectation and per rule that lost messages at the dead + * letter channel. */ public List<String> getReasons() { return Collections.unmodifiableList(reasons); } /** - * The messages that unmet expectations count as lost: the count of each unmet expectation minus the exchanges its - * route completed in time. + * The messages counted as lost: the count of each unmet expectation minus the exchanges its route completed in + * time, and the messages that a rule with {@code at=dlq} kept from reaching the dead letter channel. */ public long getLost() { return lost; @@ -161,6 +193,9 @@ public final class ChaosReport { } } sb.append(at).append(", skipped ").append(r.skipped); + if (r.lostAtDlq > 0) { + sb.append(", lost at the dead letter channel ").append(r.lostAtDlq); + } if (r.state == State.ACTIVE && r.lifecycle == null) { sb.append(", ends in ").append(ChaosRuleParser.printTime(r.endsIn)); } @@ -172,9 +207,9 @@ public final class ChaosReport { .append(ChaosRuleParser.printTime(r.within)).append(": ").append(r.outcome.key) .append(", ").append(r.completed).append(" completed"); if (r.outcome == Outcome.PENDING) { - sb.append(r.frozen - ? " (not decided: the rule was removed or changed, or the context stopped, first)" - : ", " + ChaosRuleParser.printTime(r.timeLeft) + " left"); + sb.append(", ").append(ChaosRuleParser.printTime(r.timeLeft)).append(" left"); + } else if (r.outcome == Outcome.CANCELLED) { + sb.append(" (the rule was removed or changed, or the context stopped, before it was decided)"); } } } @@ -227,7 +262,10 @@ public final class ChaosReport { */ JsonArray rulesJson(boolean currentOnly) { JsonArray ruleArray = new JsonArray(); - for (RuleEntry r : currentOnly ? rules.subList(0, current) : rules) { + for (RuleEntry r : rules) { + if (currentOnly && !r.current) { + continue; + } JsonObject jo = new JsonObject(); jo.put("id", r.id); jo.put("version", r.version); @@ -249,6 +287,9 @@ public final class ChaosReport { } jo.put("fired", fired); jo.put("skipped", r.skipped); + if (r.hooks.contains(Hook.DLQ)) { + jo.put("lostAtDlq", r.lostAtDlq); + } if (r.state == State.ACTIVE && r.lifecycle == null) { jo.put("endsIn", r.endsIn); } @@ -259,13 +300,9 @@ public final class ChaosReport { e.put("within", r.within); e.put("completed", r.completed); e.put("outcome", r.outcome.key); - if (r.outcome == Outcome.PENDING && !r.frozen) { + if (r.outcome == Outcome.PENDING) { e.put("timeLeft", r.timeLeft); } - if (r.frozen && r.outcome == Outcome.PENDING) { - // the rule was removed or changed, or the context stopped, before it was decided - e.put("notDecided", true); - } jo.put("expectation", e); } lifecycleJson(r, jo); @@ -344,6 +381,9 @@ public final class ChaosReport { /** A rule in the report. */ private static final class RuleEntry { + final ChaosRule rule; + // a current rule (not removed, replaced or moved to the history) + final boolean current; final String id; final int version; final String text; @@ -353,6 +393,7 @@ public final class ChaosReport { final long[] fired = new long[Hook.values().length]; final long firedTotal; final long skipped; + final long lostAtDlq; final long endsIn; final String expectRoute; final long expectCount; @@ -360,14 +401,15 @@ public final class ChaosReport { final Outcome outcome; final long completed; final long timeLeft; - final boolean frozen; // a lifecycle rule: its action, route, duration and what it did, else null final Action action; final String lifecycleRoute; final long duration; final ChaosLifecycle.Fault.Reading lifecycle; - RuleEntry(ChaosRule rule, long now) { + RuleEntry(ChaosRule rule, long now, boolean current) { + this.rule = rule; + this.current = current; id = rule.id; version = rule.version; text = rule.text; @@ -386,6 +428,7 @@ public final class ChaosReport { // a lifecycle rule fired when it took its route down firedTotal = total + (lifecycle != null && lifecycle.acted() ? 1 : 0); skipped = rule.skippedCount(); + lostAtDlq = rule.lostAtDlqCount(); endsIn = Math.max(0, rule.endsAt - now); within = rule.within; ChaosRule.Expectation expectation = rule.expectation; @@ -395,15 +438,13 @@ public final class ChaosReport { ChaosRule.Expectation.Reading reading = expectation.read(now); outcome = reading.outcome(); completed = reading.completed(); - timeLeft = Math.max(0, expectation.deadline - now); - frozen = reading.frozen(); + timeLeft = Math.max(0, expectation.deadline() - now); } else { expectRoute = null; expectCount = 0; outcome = null; completed = 0; timeLeft = 0; - frozen = false; } } } diff --git a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosRoutePolicyFactory.java b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosRoutePolicyFactory.java index f35d1cdf9290..0863d0be70e5 100644 --- a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosRoutePolicyFactory.java +++ b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosRoutePolicyFactory.java @@ -46,7 +46,8 @@ final class ChaosRoutePolicyFactory implements RoutePolicyFactory { /** * Changes the message of an exchange as it enters the route ({@code onExchangeBegin}); the first rule that fires * decides the fault. Counts the exchange when the route is done with it ({@code onExchangeDone}), while there are - * rules. + * rules. camel-core calls both each time the exchange goes through the route, but an exchange meets the entry rules + * of a route, and counts in it, once (see {@link ChaosExchangeState}). */ static final class ChaosRoutePolicy extends RoutePolicySupport { @@ -59,7 +60,7 @@ final class ChaosRoutePolicyFactory implements RoutePolicyFactory { @Override public void onExchangeBegin(Route route, Exchange exchange) { ChaosRule[] rules = manager.ruleSet().entry; - if (rules.length == 0) { + if (rules.length == 0 || !ChaosExchangeState.of(exchange).enter(route.getRouteId())) { return; } long now = TimeSource.currentTimeMillis(); diff --git a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosRule.java b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosRule.java index 5d2b107c120e..0ddadf31ecec 100644 --- a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosRule.java +++ b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosRule.java @@ -47,6 +47,8 @@ final class ChaosRule { /** A place where a rule can fire. */ enum Hook { SEND("send"), + // the send of the dead letter channel, which a rule reaches only with at=dlq + DLQ("dlq"), POLL("poll"), ENTRY("entry"), STEP("step"); @@ -154,16 +156,25 @@ final class ChaosRule { final Action action; // the rule says stall, not lifecycle action=suspend final boolean stall; - final long addedAt; + // when the rule was added, and its clock started: a rule added before the context started (camel.chaos.rules) + // starts its clock when the context has started (see startClock) + volatile long addedAt; + // how long the rule runs: its until duration, else camel.chaos.maxDuration + private final long runFor; // the state while it runs - final long endsAt; + volatile long endsAt; final Expectation expectation; private final AtomicLong matches = new AtomicLong(); private final LongAdder[] fired = new LongAdder[Hook.values().length]; private final LongAdder skipped = new LongAdder(); + // the messages lost because the rule failed or dropped the send of the dead letter channel, which then handled them + private final LongAdder lostAtDlq = new LongAdder(); private final AtomicLong random; private volatile State removedAs; + // the clock of the rule started: false for a rule added before the context started, until it has (guarded by the + // lock of the manager) + boolean clockStarted; // what a lifecycle rule did to its route, else null final ChaosLifecycle.Fault lifecycle; @@ -199,7 +210,8 @@ final class ChaosRule { this.stall = b.stall; this.addedAt = addedAt; // a rule without an until duration ends after camel.chaos.maxDuration (also one that ends after a count) - this.endsAt = plus(addedAt, untilDuration > 0 ? untilDuration : maxDuration); + this.runFor = untilDuration > 0 ? untilDuration : maxDuration; + this.endsAt = plus(addedAt, runFor); this.within = b.within > 0 ? b.within : (untilDuration > 0 ? untilDuration : maxDuration); this.expectation = expectRoute != null ? new Expectation(expectRoute, expectCount, plus(addedAt, within)) : null; for (int i = 0; i < fired.length; i++) { @@ -247,6 +259,27 @@ final class ChaosRule { fired[hook.ordinal()].increment(); } + /** + * Starts the clock of the rule again: its end time and the deadline of its expectation count from now. For the + * rules added before the context started, so their times do not include the startup. + */ + void startClock(long now) { + addedAt = now; + endsAt = plus(now, runFor); + if (expectation != null) { + expectation.startClock(plus(now, within)); + } + } + + /** Counts a message lost because the rule failed or dropped the send of the dead letter channel. */ + void lostAtDlq() { + lostAtDlq.increment(); + } + + long lostAtDlqCount() { + return lostAtDlq.sum(); + } + /** Counts that the rule fired at a message it could not change, so it did nothing. */ void skipped() { skipped.increment(); @@ -343,7 +376,8 @@ final class ChaosRule { /** * An expectation of a rule: the route completes at least that many more exchanges after the rule is added, before * the time is up. It is met when the count is reached in time, unmet when the time is up without it, and pending - * before that, or for good when the rule is removed (or replaced) first. + * before that. It is cancelled when the rule is removed (or replaced), or the context stops, first: the messages + * may still arrive, so it is neither met nor unmet, and does not keep the report from being final. */ static final class Expectation { @@ -351,7 +385,8 @@ final class ChaosRule { enum Outcome { MET("met"), UNMET("unmet"), - PENDING("pending"); + PENDING("pending"), + CANCELLED("cancelled"); final String key; @@ -362,7 +397,7 @@ final class ChaosRule { final String route; final long count; - final long deadline; + private long deadline; private long completed; private long metAt = -1; private long frozenAt = -1; @@ -392,6 +427,17 @@ final class ChaosRule { } } + synchronized long deadline() { + return deadline; + } + + /** The clock of the rule starts again (see {@link ChaosRule#startClock(long)}). */ + synchronized void startClock(long deadline) { + if (metAt < 0 && frozenAt < 0) { + this.deadline = deadline; + } + } + synchronized void freeze(long now) { if (frozenAt < 0) { frozenAt = now; @@ -408,7 +454,7 @@ final class ChaosRule { } if (frozenAt >= 0 && frozenAt <= deadline) { // removed before it was decided: the messages may still arrive, so they are not called lost - return Outcome.PENDING; + return Outcome.CANCELLED; } return now > deadline ? Outcome.UNMET : Outcome.PENDING; } diff --git a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosRuleParser.java b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosRuleParser.java index 35dbc0cc3342..87799bff1675 100644 --- a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosRuleParser.java +++ b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosRuleParser.java @@ -581,7 +581,7 @@ final class ChaosRuleParser { } } if (hook == null) { - throw new IllegalArgumentException("unknown at=" + at + " (the hooks are send, poll, entry and step)"); + throw new IllegalArgumentException("unknown at=" + at + " (the hooks are send, dlq, poll, entry and step)"); } if (!offered.containsKey(hook)) { throw new IllegalArgumentException( @@ -592,13 +592,16 @@ final class ChaosRuleParser { } Set<Hook> hooks = EnumSet.noneOf(Hook.class); for (Map.Entry<Hook, Set<Fault>> entry : offered.entrySet()) { - if ((b.at == null || b.at == entry.getKey()) && entry.getValue().contains(b.fault)) { + // the send of the dead letter channel is reached only with at=dlq, so endpoint=* does not lose the messages + // that the dead letter channel should keep + boolean asked = b.at == null ? entry.getKey() != Hook.DLQ : b.at == entry.getKey(); + if (asked && entry.getValue().contains(b.fault)) { hooks.add(entry.getKey()); } } if (http) { // the failure of an HTTP component is raised at the sends to it - hooks.retainAll(Set.of(Hook.SEND)); + hooks.retainAll(Set.of(Hook.SEND, Hook.DLQ)); if (hooks.isEmpty()) { throw new IllegalArgumentException( "status= is the failure of a send to an HTTP component, so it fires at the " @@ -655,6 +658,7 @@ final class ChaosRuleParser { switch (target) { case ENDPOINT, COMPONENT -> { answer.put(Hook.SEND, EnumSet.of(Fault.ERROR, Fault.DELAY, Fault.DROP, Fault.CORRUPT)); + answer.put(Hook.DLQ, EnumSet.of(Fault.ERROR, Fault.DELAY, Fault.DROP, Fault.CORRUPT)); answer.put(Hook.POLL, EnumSet.of(Fault.ERROR, Fault.DELAY, Fault.DROP)); } case ROUTE -> answer.put(Hook.ENTRY, EnumSet.of(Fault.CORRUPT)); diff --git a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosSendHook.java b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosSendHook.java index 636b47b8748a..9487bbadbb4a 100644 --- a/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosSendHook.java +++ b/components/camel-chaos/src/main/java/org/apache/camel/component/chaos/ChaosSendHook.java @@ -18,6 +18,7 @@ package org.apache.camel.component.chaos; import org.apache.camel.Endpoint; import org.apache.camel.Exchange; +import org.apache.camel.component.chaos.ChaosRule.Hook; import org.apache.camel.spi.SendHook; import org.apache.camel.util.TimeSource; @@ -25,7 +26,8 @@ import org.apache.camel.util.TimeSource; * Raises the faults of the rules at the sends to an endpoint: {@code error} (the producer is not called), {@code delay} * (the sending thread waits, through {@link TimeSource}, before it calls the producer), {@code drop} (the send reports * success without calling the producer) and {@code corrupt} (the message that is sent). The first rule that fires - * decides the fault of a send; the rules are asked in the order they were added. + * decides the fault of a send; the rules are asked in the order they were added. The send of the dead letter channel + * meets only the rules with {@code at=dlq}: the event notifier marks the exchange while the channel delivers it. */ final class ChaosSendHook implements SendHook { @@ -37,7 +39,14 @@ final class ChaosSendHook implements SendHook { @Override public long beforeSend(Exchange exchange, Endpoint endpoint) { - ChaosRule[] rules = manager.ruleSet().send; + ChaosManager.RuleSet ruleSet = manager.ruleSet(); + if (ruleSet.send.length == 0 && ruleSet.dlq.length == 0) { + return PROCEED; + } + // the send of the dead letter channel meets only the rules with at=dlq, the other sends the rest + ChaosExchangeState state = ChaosExchangeState.peek(exchange); + boolean deadLetter = state != null && state.deadLetter; + ChaosRule[] rules = deadLetter ? ruleSet.dlq : ruleSet.send; if (rules.length == 0) { return PROCEED; } @@ -45,7 +54,11 @@ final class ChaosSendHook implements SendHook { String uri = ChaosPatterns.endpointUri(endpoint); for (ChaosRule rule : rules) { if (rule.live(now) && rule.matchesUri(uri) && rule.fires(now)) { - return ChaosFaults.atSend(rule, exchange, endpoint); + if (deadLetter && (rule.fault == ChaosRule.Fault.ERROR || rule.fault == ChaosRule.Fault.DROP)) { + // the message does not reach the dead letter channel: lost, when the channel then handles it + state.deadLetterFault = rule; + } + return ChaosFaults.atSend(rule, exchange, endpoint, deadLetter ? Hook.DLQ : Hook.SEND); } } return PROCEED; diff --git a/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosReportTest.java b/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosReportTest.java index d74289b0bc13..e4333d9a291a 100644 --- a/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosReportTest.java +++ b/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosReportTest.java @@ -134,10 +134,10 @@ class ChaosReportTest extends ChaosTestSupport { entries.add(rule); } } - // the active entry first, then the replaced ones in the order they were replaced - assertThat(entries).extracting(e -> e.getInteger("version")).containsExactly(3, 1, 2); + // the versions of an id from the latest + assertThat(entries).extracting(e -> e.getInteger("version")).containsExactly(3, 2, 1); assertThat(entries).extracting(e -> e.getString("state")).containsExactly("active", "replaced", "replaced"); - assertThat(entries).extracting(e -> e.getLong("matches")).containsExactly(0L, 1L, 2L); + assertThat(entries).extracting(e -> e.getLong("matches")).containsExactly(0L, 2L, 1L); assertThat(chaos().report().toText()) .contains(id + " v3 active: endpoint=mock:other error") .contains(id + " replaced: endpoint=mock:kafka error") @@ -318,7 +318,7 @@ class ChaosReportTest extends ChaosTestSupport { } @Test - void expectationOfARemovedRuleStaysPending() { + void expectationOfARemovedRuleIsCancelled() { String id = chaos().addRule("endpoint=mock:kafka error rate=every:2 expect=orders:10 within=1m"); send(2); chaos().removeRule(id); @@ -327,12 +327,15 @@ class ChaosReportTest extends ChaosTestSupport { ChaosReport report = chaos().report(); assertThat(report.getVerdict()).isEqualTo(ChaosReport.PASS); - assertThat(report.isFinal()).isFalse(); + // a cancelled expectation does not keep the report pending + assertThat(report.isFinal()).isTrue(); JsonObject expectation = (JsonObject) ruleReport(id).get("expectation"); - assertThat(expectation.getString("outcome")).isEqualTo("pending"); - assertThat(expectation.getBoolean("notDecided")).isTrue(); + assertThat(expectation.getString("outcome")).isEqualTo("cancelled"); + assertThat(expectation.containsKey("timeLeft")).isFalse(); assertThat(expectation.getLong("completed")).isEqualTo(1); - assertThat(report.toText()).contains("(not decided: the rule was removed or changed, or the context stopped, first)"); + assertThat(report.toText()) + .contains( + "cancelled, 1 completed (the rule was removed or changed, or the context stopped, before it was decided)"); } @Test @@ -362,7 +365,31 @@ class ChaosReportTest extends ChaosTestSupport { } @Test - void anExpectationStaysPendingWhenTheContextStopsFirst() { + void resetStartsANewReport() { + String id = chaos().addRule("endpoint=nothing error expect=orders:5 within=1m"); + send(2); + clock.advance(60_001); + assertThat(chaos().report().getVerdict()).isEqualTo(ChaosReport.FAIL); + chaos().addRule("endpoint=mock:kafka error rate=every:2"); + send(2); + + assertThat(chaos().reset()).isEqualTo(2); + + ChaosReport report = chaos().report(); + assertThat(report.getVerdict()).isEqualTo(ChaosReport.PASS); + assertThat(report.isFinal()).isTrue(); + assertThat(report.getLost()).isZero(); + assertThat((JsonArray) report.toJson().get("rules")).isEmpty(); + assertThat((JsonArray) report.toJson().get("routes")).isEmpty(); + assertThat(chaos().hasRule(id)).isFalse(); + // the ids go on, so a new rule is not mistaken for an old one + assertThat(chaos().addRule("endpoint=nothing error")).isEqualTo("r3"); + send(1); + assertThat(routeCount("orders", "completed")).isEqualTo(1); + } + + @Test + void anExpectationIsCancelledWhenTheContextStopsFirst() { String id = chaos().addRule("endpoint=nothing error expect=orders:5 within=1m"); send(2); context.stop(); @@ -371,15 +398,15 @@ class ChaosReportTest extends ChaosTestSupport { ChaosReport report = chaos().report(); JsonObject expectation = (JsonObject) ruleReport(id).get("expectation"); - assertThat(expectation.getString("outcome")).isEqualTo("pending"); + assertThat(expectation.getString("outcome")).isEqualTo("cancelled"); assertThat(expectation.getLong("completed")).isEqualTo(2); assertThat(report.getVerdict()).isEqualTo(ChaosReport.PASS); - assertThat(report.isFinal()).isFalse(); + assertThat(report.isFinal()).isTrue(); assertThat(report.getLost()).isZero(); } @Test - void aPendingExpectationThatLeftTheReportKeepsItNotFinal() { + void aCancelledExpectationThatLeftTheReportKeepsItFinal() { String id = chaos().addRule("endpoint=nothing error expect=orders:5 within=1m"); chaos().removeRule(id); for (int i = 0; i < ChaosManager.MAX_HISTORY; i++) { @@ -388,6 +415,7 @@ class ChaosReportTest extends ChaosTestSupport { ChaosReport report = chaos().report(); assertThat(report.toText()).doesNotContain(id + " removed"); assertThat(report.getVerdict()).isEqualTo(ChaosReport.PASS); - assertThat(report.isFinal()).isFalse(); + // the expectation was cancelled when its rule was removed, so nothing is pending + assertThat(report.isFinal()).isTrue(); } } diff --git a/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosRouteCountsTest.java b/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosRouteCountsTest.java index bffcaa72b51c..45640c5a2052 100644 --- a/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosRouteCountsTest.java +++ b/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosRouteCountsTest.java @@ -48,6 +48,19 @@ class ChaosRouteCountsTest extends ChaosTestSupport { from("direct:wireTap").routeId("wireTap").wireTap("direct:tap").to("mock:wireTap"); from("direct:tap").routeId("tap").to("mock:tap"); from("direct:sub").routeId("sub").setHeader("sub", constant(true)).id("subStep"); + from("direct:loop").routeId("loop").loop(3).to("direct:sub").end(); + // the calling route redelivers the send to flaky, which has no error handler and fails twice + from("direct:retry").routeId("retry") + .errorHandler(defaultErrorHandler().maximumRedeliveries(2).redeliveryDelay(0)) + .to("direct:flaky"); + from("direct:flaky").routeId("flaky").errorHandler(noErrorHandler()) + .process(e -> { + int attempt = e.getProperty("attempt", 0, Integer.class) + 1; + e.setProperty("attempt", attempt); + if (attempt < 3) { + throw new IllegalStateException("attempt " + attempt); + } + }); from("direct:notHandled").routeId("notHandled") .onException(IllegalStateException.class).log("seen, not handled").end() .throwException(new IllegalStateException("not handled")); @@ -69,6 +82,55 @@ class ChaosRouteCountsTest extends ChaosTestSupport { return routeCount(route, count); } + @Test + void anExchangeThatGoesThroughARouteInALoopCountsOnce() { + String id = chaos().addRule("endpoint=nothing2:* error expect=sub:2 within=1m"); + + template.sendBody("direct:loop", "a"); + + // three passes through sub, one exchange + assertThat(count("sub", "completed")).isEqualTo(1); + assertThat(count("loop", "completed")).isEqualTo(1); + JsonObject expectation = (JsonObject) ruleReport(id).get("expectation"); + assertThat(expectation.getLong("completed")).isEqualTo(1); + assertThat(expectation.getString("outcome")).isEqualTo("pending"); + + template.sendBody("direct:loop", "b"); + assertThat(count("sub", "completed")).isEqualTo(2); + assertThat(((JsonObject) ruleReport(id).get("expectation")).getString("outcome")).isEqualTo("met"); + } + + @Test + void aRouteCalledAgainByARedeliveryCountsTheLastPass() { + template.sendBody("direct:retry", "x"); + + // two failed passes and a completed one: one completed exchange of flaky + assertThat(count("flaky", "completed")).isEqualTo(1); + assertThat(count("flaky", "failed")).isZero(); + assertThat(count("retry", "completed")).isEqualTo(1); + assertThat(count("retry", "redeliveries")).isEqualTo(2); + } + + @Test + void theEntryFaultsOfARouteAreAskedOncePerExchange() { + String id = chaos().addRule("route=sub corrupt mode=headers until=10"); + + // a header for the corrupt mode to change + template.sendBodyAndHeader("direct:loop", "a", "country", "DK"); + + assertThat(ruleReport(id).getLong("matches")).isEqualTo(1); + assertThat(fired(id, "entry")).isEqualTo(1); + } + + @Test + void aCopyOfAnExchangeCountsOnItsOwn() throws Exception { + // the original goes through sub, and so do the two copies of the multicast + template.sendBody("direct:sub", "a"); + template.sendBody("direct:multicast", "b"); + + assertThat(count("sub", "completed")).isEqualTo(3); + } + @Test void anExchangeMarkedRollbackOnlyFailsAndDoesNotMeetAnExpectation() { String id = chaos().addRule("endpoint=nothing2:* error expect=rollbackOnly:1 within=1m"); diff --git a/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosRuleParserTest.java b/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosRuleParserTest.java index 11ef2c3bb5e3..d28c7d425836 100644 --- a/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosRuleParserTest.java +++ b/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosRuleParserTest.java @@ -71,6 +71,17 @@ class ChaosRuleParserTest { assertThat(b.hooks).containsExactlyInAnyOrder(Hook.SEND, Hook.POLL); } + @Test + void theSendOfTheDeadLetterChannelNeedsAtDlq() { + assertThat(parse("endpoint=mock:dead error").hooks).doesNotContain(Hook.DLQ); + assertThat(parse("component=seda drop").hooks).doesNotContain(Hook.DLQ); + assertThat(parse("endpoint=mock:dead error at=dlq").hooks).containsExactly(Hook.DLQ); + assertThat(parse("endpoint=mock:dead delay duration=1s at=dlq").hooks).containsExactly(Hook.DLQ); + assertThat(parse("endpoint=http:dead* error status=503 at=dlq").hooks).containsExactly(Hook.DLQ); + rejected("step=s1 error at=dlq", "at=dlq is not a hook of the target step="); + rejected("route=r1 corrupt mode=headers at=dlq", "at=dlq is not a hook of the target route="); + } + @Test void errorWithoutParametersIsAChaosException() { ChaosRule.Builder b = parse("component=kafka error"); diff --git a/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosRuleTest.java b/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosRuleTest.java index 00ddcddcf241..a087d702b0fd 100644 --- a/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosRuleTest.java +++ b/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosRuleTest.java @@ -95,7 +95,7 @@ class ChaosRuleTest { Long.MAX_VALUE), "r1", 1, T0, Long.MAX_VALUE); assertThat(r.endsAt).isEqualTo(Long.MAX_VALUE); - assertThat(r.expectation.deadline).isEqualTo(Long.MAX_VALUE); + assertThat(r.expectation.deadline()).isEqualTo(Long.MAX_VALUE); assertThat(r.fires(T0)).isTrue(); assertThat(r.state(T0)).isEqualTo(State.ACTIVE); assertThat(r.expectation.outcome(T0)).isEqualTo(Outcome.PENDING); @@ -167,8 +167,8 @@ class ChaosRuleTest { removed.freeze(T0 + 50); removed.completeAt(T0 + 60); assertThat(removed.completed()).isZero(); - // removed before it was decided: pending for good, never unmet - assertThat(removed.outcome(T0 + 1000)).isEqualTo(Outcome.PENDING); + // removed before it was decided: cancelled, never unmet + assertThat(removed.outcome(T0 + 1000)).isEqualTo(Outcome.CANCELLED); ChaosRule.Expectation removedLate = new ChaosRule.Expectation("orders", 2, T0 + 100); removedLate.freeze(T0 + 200); @@ -181,7 +181,7 @@ class ChaosRuleTest { assertThat(rule("endpoint=x error until=1m expect=o:1", "r1", 1).within).isEqualTo(60_000L); assertThat(rule("endpoint=x error until=5 expect=o:1", "r1", 1).within).isEqualTo(MAX); assertThat(rule("endpoint=x error until=1m expect=o:1 within=2m", "r1", 1).within).isEqualTo(120_000L); - assertThat(rule("endpoint=x error until=1m expect=o:1 within=2m", "r1", 1).expectation.deadline) + assertThat(rule("endpoint=x error until=1m expect=o:1 within=2m", "r1", 1).expectation.deadline()) .isEqualTo(T0 + 120_000L); } } diff --git a/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosStartupClockTest.java b/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosStartupClockTest.java new file mode 100644 index 000000000000..db0b399edc66 --- /dev/null +++ b/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosStartupClockTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.chaos; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.camel.CamelContext; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.impl.DefaultCamelContext; +import org.apache.camel.util.json.JsonArray; +import org.apache.camel.util.json.JsonObject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Isolated; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The rules of {@code camel.chaos.rules} start their clock when the context has started, so a slow startup does not + * take from their until and within times. The time runs on a virtual clock (it is static, so this class runs alone). + */ +@Isolated +class ChaosStartupClockTest { + + private static final long STARTUP = TimeUnit.SECONDS.toMillis(30); + + private FakeClock clock; + private CamelContext context; + + @BeforeEach + void installClock() { + clock = FakeClock.install(); + } + + @AfterEach + void stop() { + if (context != null) { + context.stop(); + } + clock.close(); + } + + private ChaosManager startSlowly(String rules) throws Exception { + context = new DefaultCamelContext(); + Map.of("camel.chaos.enabled", "true", "camel.chaos.rules", rules) + .forEach(context.getPropertiesComponent()::addInitialProperty); + context.addRoutes(new RouteBuilder() { + @Override + public void configure() { + from("direct:start").routeId("start").to("mock:target"); + } + }); + // the rules are added when the context initializes; the startup then takes 30s + context.build(); + clock.advance(STARTUP); + context.start(); + return ChaosManager.get(context); + } + + private static JsonObject rule(ChaosManager chaos, String id) { + for (Object o : (JsonArray) chaos.report().toJson().get("rules")) { + JsonObject rule = (JsonObject) o; + if (id.equals(rule.getString("id"))) { + return rule; + } + } + throw new AssertionError("No rule " + id); + } + + @Test + void theUntilAndWithinOfAStartupRuleCountFromTheStart() throws Exception { + ChaosManager chaos = startSlowly("endpoint=mock:target error until=1m expect=start:1 within=1m"); + + JsonObject rule = rule(chaos, "r1"); + assertThat(rule.getLong("endsIn")).isEqualTo(60_000L); + assertThat(((JsonObject) rule.get("expectation")).getLong("timeLeft")).isEqualTo(60_000L); + } + + @Test + void aRuleAddedOnceTheContextRunsKeepsItsClock() throws Exception { + ChaosManager chaos = startSlowly("endpoint=nothing error"); + String id = chaos.addRule("endpoint=mock:target error until=1m"); + clock.advance(10_000); + + assertThat(rule(chaos, id).getLong("endsIn")).isEqualTo(50_000L); + // a start of the context again does not start the clock of a rule again + context.stop(); + context.start(); + assertThat(rule(chaos, id).getLong("endsIn")).isEqualTo(50_000L); + } +} diff --git a/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosStepAndErrorHandlerTest.java b/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosStepAndErrorHandlerTest.java index cd47ddeeefb1..5828497f7184 100644 --- a/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosStepAndErrorHandlerTest.java +++ b/components/camel-chaos/src/test/java/org/apache/camel/component/chaos/ChaosStepAndErrorHandlerTest.java @@ -142,16 +142,72 @@ class ChaosStepAndErrorHandlerTest extends ChaosTestSupport { } @Test - void anOpenEndpointPatternAlsoMatchesTheSendToTheDeadLetterChannel() throws Exception { + void anOpenEndpointPatternDoesNotMatchTheSendToTheDeadLetterChannel() throws Exception { String id = chaos().addRule("endpoint=mock:* error"); getMockEndpoint("mock:flaky").expectedMessageCount(0); - getMockEndpoint("mock:dead").expectedMessageCount(0); + getMockEndpoint("mock:dead").expectedMessageCount(1); template.send("direct:redeliver", e -> e.getMessage().setBody("x")); MockEndpoint.assertIsSatisfied(context); - // the first try, two redeliveries, and the send to the dead letter channel - assertThat(fired(id, "send")).isEqualTo(4); + // the first try and two redeliveries; the send of the dead letter channel needs at=dlq + assertThat(fired(id, "send")).isEqualTo(3); + assertThat(ruleReport(id).getJsonObject("fired").containsKey("dlq")).isFalse(); + assertThat(chaos().report().getLost()).isZero(); + } + + @Test + void aDlqRuleLosesTheMessageOfTheDeadLetterChannel() throws Exception { + chaos().addRule("endpoint=mock:flaky error"); + String id = chaos().addRule("endpoint=mock:dead error at=dlq"); + getMockEndpoint("mock:dead").expectedMessageCount(0); + + Exchange out = template.send("direct:redeliver", e -> e.getMessage().setBody("x")); + + MockEndpoint.assertIsSatisfied(context); + // the dead letter channel handled the new exception (deadLetterHandleNewException), without delivering it + assertThat(out.getException()).isNull(); + assertThat(fired(id, "dlq")).isEqualTo(1); + assertThat(ruleReport(id).getLong("lostAtDlq")).isEqualTo(1); + ChaosReport report = chaos().report(); + assertThat(report.getLost()).isEqualTo(1); + assertThat(report.getVerdict()).isEqualTo(ChaosReport.FAIL); + assertThat(report.getReasons()).containsExactly("rule " + id + " failed or dropped 1 send of the dead letter" + + " channel, which handled it without delivering it: 1 lost"); + assertThat(report.toText()).contains("lost at the dead letter channel 1"); + } + + @Test + void aLossAtTheDeadLetterChannelStaysInTheVerdictAfterItsRuleLeftTheReport() throws Exception { + String flaky = chaos().addRule("endpoint=mock:flaky error"); + String id = chaos().addRule("endpoint=mock:dead error at=dlq"); + template.sendBody("direct:redeliver", "x"); + chaos().removeRule(id); + chaos().removeRule(flaky); + for (int i = 0; i < ChaosManager.MAX_HISTORY; i++) { + chaos().removeRule(chaos().addRule("endpoint=nothing" + i + " error")); + } + + ChaosReport report = chaos().report(); + assertThat(report.toText()).doesNotContain(id + " removed"); + assertThat(report.getVerdict()).isEqualTo(ChaosReport.FAIL); + assertThat(report.getLost()).isEqualTo(1); + assertThat(report.getReasons()).contains( + "rules that are no longer in the report kept messages from the dead letter channel: 1 lost"); + } + + @Test + void aDlqDelayDeliversTheMessage() throws Exception { + chaos().addRule("endpoint=mock:flaky error"); + String id = chaos().addRule("endpoint=mock:dead delay duration=1ms at=dlq"); + getMockEndpoint("mock:dead").expectedMessageCount(1); + + template.sendBody("direct:redeliver", "x"); + + MockEndpoint.assertIsSatisfied(context); + assertThat(fired(id, "dlq")).isEqualTo(1); + assertThat(ruleReport(id).getLong("lostAtDlq")).isZero(); + assertThat(chaos().report().getLost()).isZero(); } @Test diff --git a/docs/user-manual/modules/ROOT/pages/camel-jbang-mcp.adoc b/docs/user-manual/modules/ROOT/pages/camel-jbang-mcp.adoc index fa7473b102fd..93b29b868044 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-jbang-mcp.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-jbang-mcp.adoc @@ -1091,7 +1091,8 @@ that changes rules is refused. The list and the report read the chaos console fr | `camel_runtime_chaos_clear` | `false` | `false` -| Removes all rules, and returns how many. +| Removes all rules, and returns how many. With `reset=true` the report starts again (no rules, route counts or + verdict), for a new experiment, as `camel cmd chaos-clear --reset` does. |=== Adding and changing a rule are marked destructive: a rule makes messages fail, wait or get lost, and its diff --git a/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-cmd-chaos-clear.adoc b/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-cmd-chaos-clear.adoc index dc110688703a..cbd9d819f5c3 100644 --- a/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-cmd-chaos-clear.adoc +++ b/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-cmd-chaos-clear.adoc @@ -19,6 +19,7 @@ camel cmd chaos-clear [options] [cols="2,5,1,2",options="header"] |=== | Option | Description | Default | Type +| `--reset` | Also start a new report: the removed rules, the counts per route and the verdict are gone | | boolean | `--timeout` | How long to wait for the integration to answer, in millis or as a duration such as 30s | 10000 | long | `-h,--help` | Display the help and sub-commands | | boolean |=== diff --git a/dsl/camel-cli-connector/src/main/java/org/apache/camel/cli/connector/ChaosAction.java b/dsl/camel-cli-connector/src/main/java/org/apache/camel/cli/connector/ChaosAction.java index aada0b4686ec..9eb8dba9b8e9 100644 --- a/dsl/camel-cli-connector/src/main/java/org/apache/camel/cli/connector/ChaosAction.java +++ b/dsl/camel-cli-connector/src/main/java/org/apache/camel/cli/connector/ChaosAction.java @@ -94,7 +94,12 @@ final class ChaosAction { answer.put("id", id); answer.put("removed", 1); } else if ("clear".equals(command)) { - answer.put("removed", call(type, chaos, "clearRules")); + // reset also starts a new report: the removed rules, the counts per route and the verdict are gone + boolean reset = root.getBooleanOrDefault("reset", false); + answer.put("removed", call(type, chaos, reset ? "reset" : "clearRules")); + if (reset) { + answer.put("reset", true); + } } else { output.error("Unknown chaos command: " + command + " (add, change, remove or clear)"); return; diff --git a/dsl/camel-cli-connector/src/test/java/org/apache/camel/cli/connector/LocalCliConnectorChaosActionTest.java b/dsl/camel-cli-connector/src/test/java/org/apache/camel/cli/connector/LocalCliConnectorChaosActionTest.java index 47e0757d99f6..45c480d5814e 100644 --- a/dsl/camel-cli-connector/src/test/java/org/apache/camel/cli/connector/LocalCliConnectorChaosActionTest.java +++ b/dsl/camel-cli-connector/src/test/java/org/apache/camel/cli/connector/LocalCliConnectorChaosActionTest.java @@ -88,9 +88,16 @@ class LocalCliConnectorChaosActionTest { /** Dispatches the action as both transports do, and returns its outcome. */ private CliActionOutcome chaos(String command, String id, String rule) throws Exception { + return chaos(command, id, rule, false); + } + + private CliActionOutcome chaos(String command, String id, String rule, boolean reset) throws Exception { JsonObject action = new JsonObject(); action.put("action", "chaos"); action.put("command", command); + if (reset) { + action.put("reset", true); + } if (id != null) { action.put("id", id); } @@ -134,7 +141,15 @@ class LocalCliConnectorChaosActionTest { assertThat(cleared.result().getInteger("removed")).isEqualTo(1); assertThat(reportRules().stream().map(JsonObject.class::cast) .map(r -> r.getString("id") + " v" + r.getInteger("version") + " " + r.getString("state"))) - .containsExactly("r1 v1 replaced", "r2 v1 removed", "r1 v2 removed"); + .containsExactly("r1 v2 removed", "r1 v1 replaced", "r2 v1 removed"); + + // a reset also starts a new report + assertThat(chaos("add", null, "step=* delay duration=1ms").result().getString("id")).isEqualTo("r3"); + CliActionOutcome reset = chaos("clear", null, null, true); + assertThat(reset.error()).isNull(); + assertThat(reset.result().getInteger("removed")).isEqualTo(1); + assertThat(reset.result().getBoolean("reset")).isTrue(); + assertThat(reportRules()).isEmpty(); } @Test diff --git a/dsl/camel-jbang/camel-jbang-core/src/generated/resources/META-INF/camel-jbang-commands-metadata.json b/dsl/camel-jbang/camel-jbang-core/src/generated/resources/META-INF/camel-jbang-commands-metadata.json index 69d3dc0e33fd..b771e638345a 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/generated/resources/META-INF/camel-jbang-commands-metadata.json +++ b/dsl/camel-jbang/camel-jbang-core/src/generated/resources/META-INF/camel-jbang-commands-metadata.json @@ -3,7 +3,7 @@ { "name": "ask", "fullName": "ask", "description": "Ask a question about a running Camel application using AI", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.Ask", "options": [ { "names": "--api-key", "description": "API key. Also reads ANTHROPIC_API_KEY, OPENAI_API_KEY, WATSONX_APIKEY, or LLM_API_KEY env vars", "javaType": "java.lang.String", "type": "string" }, { "names": "--api-type", "description": "API type: 'ollama', 'openai', 'anthropic', or 'watsonx'", "javaType": [...] { "name": "bind", "fullName": "bind", "description": "DEPRECATED: Bind source and sink Kamelets as a new Camel integration", "deprecated": true, "sourceClass": "org.apache.camel.dsl.jbang.core.commands.bind.Bind", "options": [ { "names": "--error-handler", "description": "Add error handler (none|log|sink:<endpoint>). Sink endpoints are expected in the format \"[[apigroup\/]version:]kind:[namespace\/]name\", plain Camel URIs or Kamelet name.", "javaType": "java.lang.String", "type": " [...] { "name": "catalog", "fullName": "catalog", "description": "List artifacts from Camel Catalog", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.catalog.CatalogCommand", "options": [ { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ], "subcommands": [ { "name": "component", "fullName": "catalog component", "description": "List components from the Camel Catalog", "sourceClass": "org.apache.camel.dsl.jbang.co [...] - { "name": "cmd", "fullName": "cmd", "description": "Performs commands in the running Camel integrations, such as start\/stop route, or change logging levels.", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.action.CamelAction", "options": [ { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ], "subcommands": [ { "name": "browse", "fullName": "cmd browse", "description": "Browse pending messages on endpoints [...] + { "name": "cmd", "fullName": "cmd", "description": "Performs commands in the running Camel integrations, such as start\/stop route, or change logging levels.", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.action.CamelAction", "options": [ { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ], "subcommands": [ { "name": "browse", "fullName": "cmd browse", "description": "Browse pending messages on endpoints [...] { "name": "completion", "fullName": "completion", "description": "Generate completion script for bash\/zsh", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.Complete", "options": [ { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ] }, { "name": "config", "fullName": "config", "description": "Get and set user configuration values", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.config.ConfigCommand", "options": [ { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ], "subcommands": [ { "name": "get", "fullName": "config get", "description": "Display user configuration value", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.config. [...] { "name": "debug", "fullName": "debug", "description": "Debug local Camel integration", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.Debug", "options": [ { "names": "--ago", "description": "Use ago instead of yyyy-MM-dd HH:mm:ss in timestamp.", "javaType": "boolean", "type": "boolean" }, { "names": "--background", "description": "Run in the background", "defaultValue": "false", "javaType": "boolean", "type": "boolean" }, { "names": "--background-wait", "description": "How [...] diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/CamelChaosClearAction.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/CamelChaosClearAction.java index 34f97dfd1ca1..f94c0b54be7c 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/CamelChaosClearAction.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/CamelChaosClearAction.java @@ -24,27 +24,33 @@ import picocli.CommandLine; description = "Remove all chaos rules of running integrations (camel-chaos, for development only)", sortOptions = false, showDefaultValues = true, footer = { - "%nThe rules stay in the report as removed. Without a name it acts on every running integration that", - "has chaos switched on.", + "%nThe rules stay in the report as removed; with --reset the report starts again (no rules,", + "counts or verdict). Without a name it acts on every running integration that has chaos switched on.", "%nExamples:", " camel cmd chaos-clear", - " camel cmd chaos-clear myApp" }) + " camel cmd chaos-clear myApp", + " camel cmd chaos-clear myApp --reset" }) public class CamelChaosClearAction extends CamelChaosAction { @CommandLine.Parameters(description = "Name or pid of running Camel integration", arity = "0..1") String name = "*"; + @CommandLine.Option(names = { "--reset" }, + description = "Also start a new report: the removed rules, the counts per route and the verdict are gone") + boolean reset; + public CamelChaosClearAction(CamelJBangMain main) { super(main); } @Override public Integer doCall() throws Exception { - return chaosOnEach(findChaosProcesses(name), "clear", null); + return chaosOnEach(findChaosProcesses(name), "clear", reset ? root -> root.put("reset", true) : null); } @Override protected String describe(JsonObject answer) { - return "Removed chaos rules: " + answer.get("removed"); + String removed = "Removed chaos rules: " + answer.get("removed"); + return answer.getBooleanOrDefault("reset", false) ? removed + " (report reset)" : removed; } } diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolRegistry.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolRegistry.java index 9318c3de22ce..3c1d72163917 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolRegistry.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ToolRegistry.java @@ -771,13 +771,23 @@ public final class ToolRegistry { })); register(tool("chaos_clear", - "Remove all chaos rules of the integration: no more faults, and the report keeps what they did. " - + "Returns how many were removed.") + "Remove all chaos rules of the integration: no more faults, and the report keeps what they did " + + "(with reset=true the report starts again, for a new experiment). Returns how " + + "many were removed.") .param("name", "string", CHAOS_NAME, false) + .param("reset", "boolean", + "Also start a new report: the removed rules, the counts per route and the verdict are gone", + false) .readOnly(false).destructive(false) .executor((ctx, args) -> { selectNamedProcess(ctx, args); - return chaos(ctx, "clear", null, null); + boolean reset = "true".equalsIgnoreCase(args.get("reset")); + return ctx.executeAction("chaos", root -> { + root.put("command", "clear"); + if (reset) { + root.put("reset", true); + } + }); })); } diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/process/ListChaos.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/process/ListChaos.java index f30838e954a0..aa189f0f782a 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/process/ListChaos.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/process/ListChaos.java @@ -278,10 +278,7 @@ public class ListChaos extends ProcessWatchCommand { StringBuilder sb = new StringBuilder(); sb.append(e.getString("route")).append(':').append(e.get("count")).append(' ').append(e.getString("outcome")) .append(" (").append(e.get("completed")).append(" completed"); - if (e.getBooleanOrDefault("notDecided", false)) { - // the rule was removed or changed, or the context stopped, first: it stays pending for good - sb.append(", not decided"); - } else if (e.get("timeLeft") instanceof Number n) { + if (e.get("timeLeft") instanceof Number n) { sb.append(", ").append(TimeUtils.printDuration(n.longValue())).append(" left"); } return sb.append(')').toString(); diff --git a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/process/ListChaosTest.java b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/process/ListChaosTest.java index 5323252afeae..51949221bef9 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/process/ListChaosTest.java +++ b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/process/ListChaosTest.java @@ -133,7 +133,7 @@ class ListChaosTest extends ProcessCommandTestSupport { } @Test - void reportSaysWhyAnExpectationIsNotDecided() throws Exception { + void reportShowsACancelledExpectation() throws Exception { JsonObject chaos = chaosOn(); JsonObject report = (JsonObject) chaos.get("report"); report.put("verdict", "PASS"); @@ -141,11 +141,10 @@ class ListChaosTest extends ProcessCommandTestSupport { report.put("lost", 0); report.put("reasons", new JsonArray()); JsonArray rules = report.getCollection("rules"); - // the replaced version keeps its expectation pending for good, the current one has time left + // the replaced version has its expectation cancelled, the current one has time left JsonObject replaced = (JsonObject) rules.get(1); replaced.put("expectation", new JsonObject( - Map.of("route", "orders", "count", 10, "within", 60000, "completed", 4, - "outcome", "pending", "notDecided", true))); + Map.of("route", "orders", "count", 10, "within", 60000, "completed", 4, "outcome", "cancelled"))); JsonObject current = (JsonObject) rules.get(0); current.put("expectation", new JsonObject( Map.of("route", "orders", "count", 100, "within", 60000, @@ -156,7 +155,7 @@ class ListChaosTest extends ProcessCommandTestSupport { String out = printer.getStdout(); assertThat(out).contains("PASS (not final: an expectation is pending)"); - assertThat(out).contains("orders:10 pending (4 completed, not decided)"); + assertThat(out).contains("orders:10 cancelled (4 completed)"); assertThat(out).contains("orders:100 pending (2 completed, 30s left)"); } diff --git a/dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeTools.java b/dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeTools.java index bd875f235af4..339dcf2e26eb 100644 --- a/dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeTools.java +++ b/dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeTools.java @@ -1131,10 +1131,15 @@ public class RuntimeTools { @Tool(annotations = @Tool.Annotations(readOnlyHint = false, destructiveHint = false, openWorldHint = false), description = """ Remove all chaos rules of the Camel application: no more faults, and the report keeps what they \ - did. Returns how many were removed.""") + did (with reset=true the report starts again, for a new experiment). Returns how many were \ + removed.""") public JsonObject camel_runtime_chaos_clear( - @ToolArg(description = NAME_OR_PID_DESC, required = false) String nameOrPid) { - return delegateToRegistry("chaos_clear", nameOrPid, Map.of()); + @ToolArg(description = NAME_OR_PID_DESC, required = false) String nameOrPid, + @ToolArg(description = "Also start a new report: the removed rules, the counts per route and the verdict" + + " are gone (default false)", + required = false) Boolean reset) { + return delegateToRegistry("chaos_clear", nameOrPid, + Boolean.TRUE.equals(reset) ? Map.of("reset", "true") : Map.of()); } private static void requireArg(String name, String value) { diff --git a/dsl/camel-jbang/camel-jbang-mcp/src/test/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeToolsChaosTest.java b/dsl/camel-jbang/camel-jbang-mcp/src/test/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeToolsChaosTest.java index c2b3e8fe9598..435a134deff2 100644 --- a/dsl/camel-jbang/camel-jbang-mcp/src/test/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeToolsChaosTest.java +++ b/dsl/camel-jbang/camel-jbang-mcp/src/test/java/org/apache/camel/dsl/jbang/core/commands/mcp/RuntimeToolsChaosTest.java @@ -183,7 +183,7 @@ class RuntimeToolsChaosTest { assertThat(rules(awaitList(l -> rules(l).size() == 1))) .containsExactly("r1 v2 endpoint=direct:target delay duration=1ms"); - assertThat(tools.camel_runtime_chaos_clear("shop").getInteger("removed")).isEqualTo(1); + assertThat(tools.camel_runtime_chaos_clear("shop", null).getInteger("removed")).isEqualTo(1); assertThat(rules(awaitList(l -> rules(l).isEmpty()))).isEmpty(); // the report keeps every version of every rule @@ -196,7 +196,14 @@ class RuntimeToolsChaosTest { assertThat(report[0].<Map<String, Object>> getMap("report")).containsKey("verdict"); assertThat(((List<?>) report[0].getMap("report").get("rules")).stream().map(Map.class::cast) .map(r -> r.get("id") + " v" + r.get("version") + " " + r.get("state"))) - .containsExactly("r1 v1 replaced", "r2 v1 removed", "r1 v2 removed"); + .containsExactly("r1 v2 removed", "r1 v1 replaced", "r2 v1 removed"); + + // reset=true also starts a new report + JsonObject reset = tools.camel_runtime_chaos_clear("shop", true); + assertThat(reset.getInteger("removed")).isZero(); + assertThat(reset.getBoolean("reset")).isTrue(); + awaitThat("the report without rules", () -> tools.camel_runtime_chaos_report("shop").getMap("report") + .get("rules") instanceof List<?> l && l.isEmpty()); } @Test @@ -254,7 +261,7 @@ class RuntimeToolsChaosTest { () -> tools.camel_runtime_chaos_add("shop", RULE_ERROR), () -> tools.camel_runtime_chaos_change("shop", "r1", RULE_ERROR), () -> tools.camel_runtime_chaos_remove("shop", "r1"), - () -> tools.camel_runtime_chaos_clear("shop"))) { + () -> tools.camel_runtime_chaos_clear("shop", null))) { assertThatThrownBy(change::run) .isInstanceOf(ToolCallException.class) .hasMessageStartingWith("Chaos is off in this integration"); @@ -303,7 +310,7 @@ class RuntimeToolsChaosTest { assertThatThrownBy(() -> tools.camel_runtime_chaos_add("orders", RULE_ERROR)) .isInstanceOf(ToolCallException.class) .hasMessageStartingWith("No unique Camel process found matching 'orders'"); - assertThatThrownBy(() -> tools.camel_runtime_chaos_clear(null)) + assertThatThrownBy(() -> tools.camel_runtime_chaos_clear(null, null)) .isInstanceOf(ToolCallException.class) .hasMessageStartingWith("Multiple Camel processes running"); assertThatThrownBy(() -> tools.camel_runtime_chaos_list("billing"))
