This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch tui-chaos-explore-start in repository https://gitbox.apache.org/repos/asf/camel.git
commit 86d579d4dd85daf534ba13eefd2259d089d1661f Author: Claus Ibsen <[email protected]> AuthorDate: Tue Nov 10 17:27:57 2026 +0100 chore: camel-jbang - let there be chaos from the chaos screen (L): pick the rules to try, one experiment each L on the Chaos tab opens the start of an exploration of the integration: the rules it suggests, the most worth trying first, each with what should hold, and a budget of 2 to 30 minutes. The TUI tries the picked rules one at a time, each for its share of the time left, and the Explore view says which rule is tried; x stops it. No AI: that is a follow-up. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../ROOT/pages/camel-jbang-tui-observe.adoc | 10 + .../dsl/jbang/core/commands/tui/ChaosControl.java | 127 ++++++++- .../core/commands/tui/ChaosExploreDialog.java | 306 +++++++++++++++++++++ .../jbang/core/commands/tui/ChaosExploreRun.java | 264 ++++++++++++++++++ .../jbang/core/commands/tui/ChaosExploreView.java | 19 +- .../dsl/jbang/core/commands/tui/ChaosTab.java | 53 +++- .../dsl/jbang/core/commands/tui/DialogHelper.java | 11 +- .../src/main/resources/tui/help/chaos.md | 18 ++ .../jbang/core/commands/tui/ChaosExploreTest.java | 281 +++++++++++++++++++ 9 files changed, 1071 insertions(+), 18 deletions(-) diff --git a/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc b/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc index 7ae636b98e7e..eb9fb55ead48 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc @@ -393,6 +393,16 @@ hypothesis and the evidence in a few words, and the selected one in detail: its routes did, the messages lost at a dead letter channel, and why it could not decide when it could not (such as faults that never fired, or a route that had no messages). *l* or Esc shows the rules again. +Press *L* to start an exploration by hand, on the integration you look at: pick the rules to try from what the +integration suggests, the most worth trying first, each with what should hold despite its faults, and a budget of 2 to +30 minutes. The single faults not tried yet are picked, of the routes that get messages by themselves (a timer, or a +route they are called from); a route whose messages come from outside is marked, as its rule says nothing without +messages. *Space* picks a rule, *Left* and *Right* change the budget, and *Enter* starts. The picked rules are tried one +at a time, each for its share of the budget, and each ends with PASS, FAIL or INCONCLUSIVE, while the line above the +experiments says which rule is tried. *x* stops it, which removes the rules at once. Every rule is removed when the +budget ends, also when the TUI is closed. For a longer exploration, where an AI picks what to try next, use +`camel cmd chaos-explore`. + image::jbang/camel-tui-chaos-explore.png[The exploration of the chaos example on the Chaos tab: six experiments in five minutes on a strip of time, and the failed one in detail: 2 orders lost at the dead letter channel] === CVE Audit diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosControl.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosControl.java index 46bdcf396108..45442bd1b7de 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosControl.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosControl.java @@ -62,6 +62,8 @@ final class ChaosControl { RECIPES, // what to try, the rules the integration suggests (null while it is asked) SUGGEST, + // the start of an exploration: the rules to try and the budget (null while the integration is asked) + EXPLORE, INPUT, CONFIRM, RESULT @@ -90,6 +92,10 @@ final class ChaosControl { private boolean recipesLoaded; private ChaosRecipeForm form; private volatile ChaosSuggestList suggestions; + private volatile ChaosExploreDialog exploreDialog; + // the exploration started here (L), and whether one was just started, for the screen to show it + private final ChaosExploreRun explorer; + private boolean exploreStarted; // set by the background task that sends the action, read when the screen is drawn private volatile Step step = Step.NONE; private Command command; @@ -116,9 +122,30 @@ final class ChaosControl { } ChaosControl(MonitorContext ctx, Function<JsonObject, JsonObject> sender, Supplier<ChaosRecipes> recipeSource) { + this(ctx, sender, recipeSource, new ChaosExploreRun(ctx)); + } + + ChaosControl(MonitorContext ctx, Function<JsonObject, JsonObject> sender, Supplier<ChaosRecipes> recipeSource, + ChaosExploreRun explorer) { this.ctx = ctx; this.sender = sender; this.recipeSource = recipeSource; + this.explorer = explorer; + } + + ChaosExploreRun explorer() { + return explorer; + } + + ChaosExploreDialog exploreDialog() { + return exploreDialog; + } + + /** Whether an exploration was started since the last call, so the screen shows it. */ + boolean takeExploreStarted() { + boolean started = exploreStarted; + exploreStarted = false; + return started; } ChaosRecipeForm form() { @@ -131,7 +158,7 @@ final class ChaosControl { boolean isTextInputActive() { // the list of known rules takes the keys too, so a digit or a letter does not leave the screen - return step == Step.INPUT || step == Step.RECIPES || step == Step.SUGGEST; + return step == Step.INPUT || step == Step.RECIPES || step == Step.SUGGEST || step == Step.EXPLORE; } boolean isConfirming() { @@ -157,6 +184,9 @@ final class ChaosControl { case SUGGEST -> { return handleSuggestKey(ke); } + case EXPLORE -> { + return handleExploreKey(ke); + } case INPUT -> { return handleInputKey(ke); } @@ -323,6 +353,78 @@ final class ChaosControl { return true; } + /** + * Let there be chaos (CAMEL-26760): opens the start of an exploration of the integration, with the rules it + * suggests to pick from, or says why it cannot start. + */ + void openExplore(IntegrationInfo info) { + if (info == null || info.chaos == null || !info.chaos.enabled) { + return; + } + pid = info.pid; + name = info.name; + id = null; + rule = null; + command = Command.ADD; + if (explorer.isRunning(info.pid) + || info.chaos.exploration != null && info.chaos.exploration.running()) { + // one exploration at a time, whoever started it + resultLines.clear(); + result("Let there be chaos", false, "An exploration runs in " + name + " already: l shows it"); + return; + } + exploreDialog = null; + step = Step.EXPLORE; + JsonObject request = new JsonObject(); + request.put("action", "chaos"); + request.put("command", "suggest"); + String target = pid; + String of = name; + Runnable task = () -> { + JsonObject reply = sender != null ? sender.apply(request) : ctx.executeAction(target, request, 5000); + if (step != Step.EXPLORE) { + // closed while it was asked + return; + } + if (reply == null || reply.getString("error") != null) { + step = Step.NONE; + resultLines.clear(); + result("Let there be chaos", true, reply == null + ? "The integration " + of + " did not answer" + : reply.getString("error")); + return; + } + exploreDialog = new ChaosExploreDialog(of, reply); + }; + if (sender != null) { + task.run(); + } else { + ctx.backgroundExecutor.execute(task); + } + } + + private boolean handleExploreKey(KeyEvent ke) { + ChaosExploreDialog dialog = exploreDialog; + if (dialog == null) { + if (ke.isCancel()) { + close(); + } + return true; + } + switch (dialog.handleKey(ke)) { + case CLOSED -> close(); + case START -> { + explorer.start(pid, dialog.picked(), dialog.budget(), dialog.budgetMillis()); + exploreStarted = true; + close(); + } + default -> { + // the dialog used the key + } + } + return true; + } + /** Opens a rule typed as one line, to change before Enter: a rule that makes a real error again (CAMEL-26723). */ void openRule(IntegrationInfo info, String text) { if (!ChaosHere.available(info)) { @@ -462,7 +564,7 @@ final class ChaosControl { // the form goes back to the list, the list closes return handleRecipeKey(KeyEvent.ofKey(KeyCode.ESCAPE)); } - if (step == Step.SUGGEST) { + if (step == Step.SUGGEST || step == Step.EXPLORE) { close(); return true; } @@ -475,6 +577,7 @@ final class ChaosControl { input = null; form = null; suggestions = null; + exploreDialog = null; describing = null; } @@ -603,7 +706,16 @@ final class ChaosControl { list.render(frame, area); } else { DialogHelper.renderConfirm(frame, area, "What to Try", "Asking " + name + " what to try...", null, - "Esc", "cancel", null, null); + null, null, null, null); + } + } + case EXPLORE -> { + ChaosExploreDialog dialog = exploreDialog; + if (dialog != null) { + dialog.render(frame, area); + } else { + DialogHelper.renderConfirm(frame, area, "Let there be chaos", "Asking " + name + " what to try...", + null, null, null, null, null); } } case INPUT -> { @@ -704,6 +816,14 @@ final class ChaosControl { TuiHelper.hint(spans, "Esc", "cancel"); } } + case EXPLORE -> { + ChaosExploreDialog dialog = exploreDialog; + if (dialog != null) { + dialog.renderFooter(spans); + } else { + TuiHelper.hint(spans, "Esc", "cancel"); + } + } case INPUT -> { TuiHelper.hint(spans, "Enter", command == Command.DESCRIBE ? "ask the AI" : command == Command.SAVE ? "save" : "confirm"); @@ -725,6 +845,7 @@ final class ChaosControl { TuiHelper.hint(spans, "s", "save as scenario"); } TuiHelper.hint(spans, "R", "new experiment"); + TuiHelper.hint(spans, "L", "let there be chaos"); if (ctx.askAiCallback != null) { TuiHelper.hint(spans, "d", "describe with AI"); if (hasRules) { diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosExploreDialog.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosExploreDialog.java new file mode 100644 index 000000000000..6098e30fd15d --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosExploreDialog.java @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands.tui; + +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import dev.tamboui.layout.Constraint; +import dev.tamboui.layout.Layout; +import dev.tamboui.layout.Rect; +import dev.tamboui.style.Overflow; +import dev.tamboui.style.Style; +import dev.tamboui.terminal.Frame; +import dev.tamboui.text.Line; +import dev.tamboui.text.Span; +import dev.tamboui.text.Text; +import dev.tamboui.tui.event.KeyCode; +import dev.tamboui.tui.event.KeyEvent; +import dev.tamboui.widgets.Clear; +import dev.tamboui.widgets.block.Block; +import dev.tamboui.widgets.block.BorderType; +import dev.tamboui.widgets.block.Borders; +import dev.tamboui.widgets.paragraph.Paragraph; +import org.apache.camel.util.TimeUtils; +import org.apache.camel.util.json.JsonObject; + +/** + * Let there be chaos ({@code L} on the Chaos screen, CAMEL-26760): the start of an exploration of the integration, by + * hand. The rules to try are the suggestions of the integration, the most worth trying first, each with what should + * hold despite its faults; the ones worth trying now are picked. The exploration tries the picked rules one at a time, + * each for its share of the budget (see {@link ChaosExploreRun}). + */ +final class ChaosExploreDialog { + + /** What a key did to the dialog. */ + enum Outcome { + NONE, + CLOSED, + START + } + + /** + * A suggestion to try. + * + * @param rule its rules, separated by ; for two faults at once + * @param expect what should hold, such as orders:failed<=0 + * @param fed whether its route gets messages without a feed (a timer, or a route it is called from) + */ + record Item(String route, String kind, String title, String rule, String why, String expect, boolean tried, + boolean fed) { + + boolean combined() { + return "combined".equals(kind); + } + + boolean deadLetter() { + return "dead letter channel".equals(kind); + } + } + + /** The budgets to pick from: an exploration by hand is short, the long ones are for camel cmd chaos-explore. */ + static final List<String> BUDGETS = List.of("2m", "5m", "10m", "15m", "30m"); + // the shortest time a rule is tried, so its faults fire and the routes have messages + static final long MIN_SHARE = 30_000; + // the rules picked when the dialog opens: single faults, then two at once + private static final int PICKED_SINGLE = 3; + private static final int PICKED_COMBINED = 1; + private static final Pattern EXPECT = Pattern.compile("expect=(\\S+)"); + + private final String name; + private final List<Item> items = new ArrayList<>(); + private final List<Boolean> picked = new ArrayList<>(); + private int selected; + private int budget = BUDGETS.indexOf("10m"); + + ChaosExploreDialog(String name, JsonObject reply) { + this.name = name; + if (reply != null && reply.getCollection("suggestions") != null) { + for (Object o : reply.getCollection("suggestions")) { + if (o instanceof JsonObject jo && jo.getString("rule") != null) { + // two faults at once (CAMEL-26772): the rules are added together, separated by ; + List<Object> rules = jo.getCollection("rules"); + String rule = rules != null && !rules.isEmpty() + ? rules.stream().map(String::valueOf).collect(Collectors.joining("; ")) + : jo.getString("rule"); + Matcher m = EXPECT.matcher(rule); + JsonObject feed = jo.getMap("feed"); + items.add(new Item( + jo.getString("route"), jo.getString("kind"), jo.getString("title"), rule, jo.getString("why"), + m.find() ? m.group(1) : null, jo.getBooleanOrDefault("tried", false), + feed != null && feed.getBooleanOrDefault("fed", false))); + } + } + } + // the faults not tried yet whose routes get messages by themselves, the most worth trying first: a few single + // ones, then two at once, as most bugs that lose messages need two failures that meet; a route without messages + // says nothing, and a dead letter channel fails only when another fault sends it a message (the pair does) + int singles = 0; + int combined = 0; + for (Item item : items) { + boolean worth = !item.tried() && item.fed() && item.expect() != null && !item.deadLetter(); + boolean pick = worth && (item.combined() ? combined < PICKED_COMBINED : singles < PICKED_SINGLE); + picked.add(pick); + if (pick && item.combined()) { + combined++; + } else if (pick) { + singles++; + } + } + } + + List<Item> items() { + return items; + } + + /** The picked rules, in the order of the list. */ + List<Item> picked() { + List<Item> answer = new ArrayList<>(); + for (int i = 0; i < items.size(); i++) { + if (picked.get(i)) { + answer.add(items.get(i)); + } + } + return answer; + } + + String budget() { + return BUDGETS.get(budget); + } + + long budgetMillis() { + return TimeUtils.toMilliSeconds(budget()); + } + + /** The time each picked rule is tried, or 0 when none is picked. */ + long share() { + int count = picked().size(); + return count == 0 ? 0 : budgetMillis() / count; + } + + /** Why the exploration cannot start, or null when it can. */ + String cannotStart() { + if (picked().isEmpty()) { + return "Pick a rule to try (Space)"; + } + if (share() < MIN_SHARE) { + return "Too many rules for " + budget() + ": pick fewer, or a longer budget"; + } + return null; + } + + Outcome handleKey(KeyEvent ke) { + if (ke.isCancel()) { + return Outcome.CLOSED; + } + if (ke.isUp() && selected > 0) { + selected--; + } else if (ke.isDown() && selected < items.size() - 1) { + selected++; + } else if (ke.isKey(KeyCode.LEFT) && budget > 0) { + budget--; + } else if (ke.isKey(KeyCode.RIGHT) && budget < BUDGETS.size() - 1) { + budget++; + } else if (ke.isChar(' ') && !items.isEmpty()) { + picked.set(selected, !picked.get(selected)); + } else if (ke.isConfirm() && cannotStart() == null) { + return Outcome.START; + } + return Outcome.NONE; + } + + void render(Frame frame, Rect area) { + int width = Math.min(area.width() - 4, 150); + int height = Math.min(area.height() - 2, Math.max(16, items.size() + 10)); + Rect popup = DialogHelper.centered(area, width, height); + frame.renderWidget(Clear.INSTANCE, popup); + Block outer = Block.builder().borderType(BorderType.ROUNDED).borders(Borders.ALL).borderStyle(Theme.warning()) + .title(" Let there be chaos: " + name + " ").build(); + frame.renderWidget(outer, popup); + Rect inner = outer.inner(popup); + if (items.isEmpty()) { + frame.renderWidget(Paragraph.builder().overflow(Overflow.WRAP_WORD).text(Text.from(List.of(Line.empty(), + Line.from(Span.styled(" Nothing to try: the routes call nothing remote, and have no input from" + + " outside, dead letter channel or hand-off to another route", + Theme.muted()))))) + .build(), + inner); + return; + } + List<Rect> rows = Layout.vertical() + .constraints(Constraint.length(2), Constraint.fill(), Constraint.length(3)) + .split(inner); + renderBudget(frame, rows.get(0)); + List<Rect> cols = Layout.horizontal() + .constraints(Constraint.length(Math.min(76, rows.get(1).width() / 2)), Constraint.fill()) + .split(rows.get(1)); + renderList(frame, cols.get(0)); + renderDetail(frame, cols.get(1)); + String why = cannotStart(); + frame.renderWidget(Paragraph.builder().overflow(Overflow.WRAP_WORD).text(Text.from( + Line.from(Span.styled(why != null + ? " " + why + : " The rules are tried one at a time on " + name + ", each ends with PASS or FAIL. Every rule" + + " is removed when the budget ends, and the routes are not changed.", + why != null ? Theme.warning() : Theme.muted())))) + .block(Block.builder().borders(EnumSet.of(Borders.TOP)).borderStyle(Theme.border()).build()).build(), + rows.get(2)); + } + + private void renderBudget(Frame frame, Rect area) { + int count = picked().size(); + List<Span> spans = new ArrayList<>(); + spans.add(Span.styled(" Budget ", Theme.label())); + spans.add(Span.styled("◂ " + budget() + " ▸", Style.EMPTY.bold())); + spans.add(Span.styled(count == 0 + ? " no rule picked" + : " " + count + (count == 1 ? " rule" : " rules") + ", " + ChaosTab.shortTime(share()) + " each", + Theme.muted())); + frame.renderWidget(Paragraph.builder().text(Text.from(Line.from(spans))).build(), area); + } + + private void renderList(Frame frame, Rect area) { + List<Line> lines = new ArrayList<>(); + lines.add(Line.from(Span.styled(" Rules to try, the most worth trying first", Style.EMPTY.bold()))); + int selectedLine = 0; + for (int i = 0; i < items.size(); i++) { + Item item = items.get(i); + boolean sel = i == selected; + if (sel) { + selectedLine = lines.size(); + } + Style style = sel ? Theme.selectionBg().bold() : Style.EMPTY; + List<Span> spans = new ArrayList<>(); + spans.add(Span.styled((sel ? " > " : " ") + (picked.get(i) ? "[x] " : "[ ] ") + item.title(), style)); + spans.add(Span.styled(" " + item.route(), Theme.muted())); + if (item.tried()) { + spans.add(Span.styled(" tried", Theme.success())); + } + if (!item.fed()) { + spans.add(Span.styled(" needs messages", Theme.warning())); + } + lines.add(Line.from(spans)); + } + // keep the selected one in sight + int scroll = Math.max(0, selectedLine - (area.height() - 2)); + frame.renderWidget(Paragraph.builder().text(Text.from(lines)).scroll(scroll) + .block(Block.builder().borders(EnumSet.of(Borders.RIGHT)).borderStyle(Theme.border()).build()).build(), area); + } + + private void renderDetail(Frame frame, Rect area) { + Item item = items.get(selected); + List<Line> lines = new ArrayList<>(); + lines.add(Line.from(Span.styled(" " + item.title(), Style.EMPTY.bold()), + Span.styled(" route " + item.route() + " · " + item.kind(), Theme.muted()))); + lines.add(Line.empty()); + lines.add(Line.from(Span.raw(" " + item.why()))); + lines.add(Line.empty()); + lines.add(Line.from(Span.styled(" " + item.rule(), Theme.warning()))); + if (item.expect() != null) { + lines.add(Line.empty()); + lines.add(Line.from(Span.styled(" Should hold: ", Theme.label()), Span.raw(item.expect()))); + } + if (item.tried()) { + lines.add(Line.empty()); + lines.add(Line.from(Span.styled(" A rule of this kind was tried in route " + item.route() + " already", + Theme.success()))); + } + if (!item.fed()) { + lines.add(Line.empty()); + lines.add(Line.from(Span.styled(" Route " + item.route() + " gets its messages from outside: send it some" + + " while the rule runs, or the verdict says nothing (INCONCLUSIVE)", + Theme.warning()))); + } + frame.renderWidget(Paragraph.builder().overflow(Overflow.WRAP_WORD).text(Text.from(lines)).build(), area); + } + + void renderFooter(List<Span> spans) { + if (!items.isEmpty()) { + TuiHelper.hint(spans, "↑↓", "choose"); + TuiHelper.hint(spans, "Space", "pick"); + TuiHelper.hint(spans, "←→", "budget"); + if (cannotStart() == null) { + TuiHelper.hint(spans, "Enter", "start"); + } + } + TuiHelper.hint(spans, "Esc", "cancel"); + } +} diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosExploreRun.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosExploreRun.java new file mode 100644 index 000000000000..53cc8b7708a2 --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosExploreRun.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands.tui; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.apache.camel.dsl.jbang.core.common.ChaosExperimentRun; +import org.apache.camel.dsl.jbang.core.common.RuntimeHelper; +import org.apache.camel.util.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An exploration started from the Chaos screen (let there be chaos, CAMEL-26760): the picked rules are tried one at a + * time, each as an experiment for its share of the budget, and the exploration is finished at the end. It runs on a + * thread of its own, as an experiment waits for its verdict; camel-chaos enforces the budget, so its rules end at the + * deadline also when the TUI is gone. The Explore view shows the experiments as the integration reports them, and this + * says which rule is being tried. + */ +final class ChaosExploreRun { + + private static final Logger LOG = LoggerFactory.getLogger(ChaosExploreRun.class); + private static final Pattern UNTIL = Pattern.compile("\\buntil=\\S+"); + private static final Pattern WITHIN = Pattern.compile("\\bwithin=\\S+"); + // the order of the verdicts in the summary + private static final List<String> VERDICTS = List.of("PASS", "FAIL", "INCONCLUSIVE", "REFUSED"); + // left over at the deadline, for the last experiment to end before camel-chaos ends it + private static final long MARGIN = 3000; + // how much longer than its share an experiment may wait for its verdict + private static final long GRACE = 5000; + + // the chaos action and the status of the integration with the pid, for the experiments + private final Function<String, ChaosExperimentRun.Integration> integrations; + private Thread thread; + // set on the thread of the run, read when the screen is drawn + private volatile String pid; + private volatile List<ChaosExploreDialog.Item> picks = List.of(); + private volatile int current; + private volatile long share; + private volatile long deadline; + private volatile long ruleStarted; + private volatile boolean running; + private volatile boolean stopped; + private volatile String outcome; + private final Map<String, Integer> verdicts = new TreeMap<>( + (a, b) -> Integer.compare(order(a), order(b)) != 0 ? Integer.compare(order(a), order(b)) : a.compareTo(b)); + + ChaosExploreRun(MonitorContext ctx) { + this(pid -> integration(ctx, pid)); + } + + ChaosExploreRun(Function<String, ChaosExperimentRun.Integration> integrations) { + this.integrations = integrations; + } + + /** Whether an exploration started here runs in the integration with the pid. */ + boolean isRunning(String pid) { + return running && pid != null && pid.equals(this.pid); + } + + /** Starts the exploration: the picked rules one at a time, each for its share of the budget. */ + synchronized void start(String pid, List<ChaosExploreDialog.Item> picks, String budget, long budgetMillis) { + if (running) { + return; + } + this.pid = pid; + this.picks = List.copyOf(picks); + this.share = budgetMillis / Math.max(1, picks.size()); + this.deadline = MonotonicClock.millis() + budgetMillis; + this.current = 0; + this.stopped = false; + this.outcome = null; + synchronized (verdicts) { + verdicts.clear(); + } + running = true; + thread = new Thread(() -> run(budget), "camel-tui-chaos-explore"); + thread.setDaemon(true); + thread.start(); + } + + /** Stops the exploration: it is finished now, which removes its rules. */ + synchronized void stop() { + if (!running) { + return; + } + stopped = true; + if (thread != null) { + // the experiment that waits for its verdict is ended, and the exploration finished, on the way out + thread.interrupt(); + } + } + + /** Runs the exploration, on its own thread. */ + void run(String budget) { + ChaosExperimentRun.Integration integration = integrations.apply(pid); + try { + integration.chaos("explore-start", root -> root.put("budget", budget)); + } catch (RuntimeException e) { + outcome = "Not started: " + e.getMessage(); + running = false; + return; + } + try { + for (int i = 0; i < picks.size() && !stopped; i++) { + current = i; + ruleStarted = MonotonicClock.millis(); + // the time left, shared by the rules left: an experiment takes a little longer than its rules run + share = Math.max(1000, (deadline - ruleStarted - MARGIN) / (picks.size() - i)); + ChaosExploreDialog.Item pick = picks.get(i); + String verdict; + try { + JsonObject ended = ChaosExperimentRun.run(integration, pick.route() + ": " + pick.title(), + forShare(pick.rule(), share), null, share + GRACE, null); + verdict = ended.getString("verdict"); + if (ended.getLongOrDefault("timeLeft", 1) <= 0) { + // the deadline of the exploration: camel-chaos has ended it + count(verdict); + break; + } + } catch (IllegalStateException e) { + // interrupted by stop + break; + } catch (RuntimeException e) { + // the integration refused it, such as a rule that is not valid there: the next one + LOG.debug("Chaos experiment {} refused: {}", pick.rule(), e.getMessage()); + verdict = "REFUSED"; + } + count(verdict); + } + } finally { + // a stop interrupted the wait for the verdict: the finish is still sent + Thread.interrupted(); + try { + integration.chaos("explore-finish", root -> { + }); + } catch (RuntimeException e) { + // finished already, at its deadline + LOG.debug("Chaos exploration finished already: {}", e.getMessage()); + } + int tried = tried(); + outcome = (stopped ? "Stopped after " : "Done: ") + tried + (tried == 1 ? " rule" : " rules") + " tried" + + (tried == 0 ? "" : ", " + verdictCounts()); + running = false; + } + } + + private static int order(String verdict) { + int i = VERDICTS.indexOf(verdict); + return i >= 0 ? i : VERDICTS.size(); + } + + private void count(String verdict) { + synchronized (verdicts) { + verdicts.merge(verdict != null ? verdict : "INCONCLUSIVE", 1, Integer::sum); + } + } + + private int tried() { + synchronized (verdicts) { + return verdicts.values().stream().mapToInt(Integer::intValue).sum(); + } + } + + private String verdictCounts() { + synchronized (verdicts) { + return verdicts.entrySet().stream().map(e -> e.getValue() + " " + e.getKey()) + .collect(Collectors.joining(", ")); + } + } + + /** + * What the exploration started here is doing, for the Explore view of the integration with the pid: the rule it + * tries, or how it ended. Null when none was started here for it. + */ + String progress(String pid) { + if (pid == null || !pid.equals(this.pid)) { + return null; + } + if (!running) { + return outcome; + } + if (stopped) { + return "Stopping: the rules are removed"; + } + List<ChaosExploreDialog.Item> list = picks; + int i = current; + if (i >= list.size()) { + return null; + } + ChaosExploreDialog.Item pick = list.get(i); + long left = share - (MonotonicClock.millis() - ruleStarted); + return "Rule " + (i + 1) + " of " + list.size() + ": " + pick.route() + ", " + pick.title() + + (left > 0 + ? ", " + ChaosTab.countdown(left) + " left of " + ChaosTab.shortTime(share) + : ", waiting for the verdict") + + (tried() == 0 ? "" : " · " + verdictCounts()); + } + + /** + * The rules for their share of the budget: each ends when its share is up ({@code until=}), and an expectation that + * waits longer for its messages ({@code within=}) waits that long at most. + */ + static String forShare(String rules, long share) { + String seconds = Math.max(1, share / 1000) + "s"; + return Arrays.stream(rules.split(";")).map(String::strip).filter(r -> !r.isEmpty()).map(r -> { + String rule = UNTIL.matcher(r).find() ? UNTIL.matcher(r).replaceAll("until=" + seconds) : r + " until=" + seconds; + return WITHIN.matcher(rule).replaceAll("within=" + seconds); + }).collect(Collectors.joining("; ")); + } + + /** The integration with the pid, as the TUI reaches it: its chaos action, and its status file. */ + private static ChaosExperimentRun.Integration integration(MonitorContext ctx, String pid) { + return new ChaosExperimentRun.Integration() { + @Override + public JsonObject chaos(String command, Consumer<JsonObject> options) { + JsonObject request = new JsonObject(); + request.put("action", "chaos"); + request.put("command", command); + options.accept(request); + JsonObject reply = ctx.executeAction(pid, request, 10000); + if (reply == null) { + throw new IllegalArgumentException("The integration did not answer"); + } + if (reply.getString("error") != null) { + throw new IllegalArgumentException(reply.getString("error")); + } + return reply; + } + + @Override + public JsonObject send(ChaosExperimentRun.Feed feed) { + // the exploration of the TUI sends no messages: it tries the rules of routes that get them by themselves + throw new UnsupportedOperationException("No feed"); + } + + @Override + public JsonObject status() { + return RuntimeHelper.readStatus(Long.parseLong(pid)); + } + }; + } +} diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosExploreView.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosExploreView.java index df2761924eb2..093fa99fb6df 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosExploreView.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosExploreView.java @@ -57,16 +57,25 @@ final class ChaosExploreView { /** * Renders the exploration. * - * @return the area of the table of experiments, for the clicks + * @param progress what the exploration started on this screen is doing (L), or null + * @return the area of the table of experiments, for the clicks */ - static Rect render(Frame frame, Rect area, ChaosInfo.Exploration ex, TableState state, String name) { + static Rect render( + Frame frame, Rect area, ChaosInfo.Exploration ex, TableState state, String name, String progress) { + if (progress != null) { + List<Rect> rows = Layout.vertical().constraints(Constraint.length(1), Constraint.fill()).split(area); + frame.renderWidget(Paragraph.builder() + .text(Text.from(Line.from(Span.styled(" 🎲 " + progress, Theme.warning())))).build(), rows.get(0)); + area = rows.get(1); + } if (ex == null) { frame.renderWidget(Paragraph.builder() .text(Text.from(Line.from(Span.raw("")), - Line.from(Span.styled(" No exploration yet. Let there be chaos: an AI tries faults on the" - + " integration by itself for a budget of time, and reports", + Line.from(Span.styled(" No exploration yet. Let there be chaos: press L to pick the rules to" + + " try, and they are tried one at a time for a budget of time.", Theme.muted())), - Line.from(Span.styled(" what held and what broke: camel cmd chaos-explore " + Line.from(Span.styled(" Or let an AI try faults by itself, and report what held and what" + + " broke: camel cmd chaos-explore " + (name != null ? name : "<name>") + " --budget=10m", Theme.muted())))) .block(block(" Explore ")) diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosTab.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosTab.java index b9733e122b76..8a02e7cdc47b 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosTab.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosTab.java @@ -67,6 +67,8 @@ class ChaosTab extends AbstractTableTab { // the exploration instead of the rules (l, let there be chaos), and the selected rule meanwhile private boolean explore; private Integer rulesSelection; + // the selection follows the experiment that runs, of an exploration started here, until another one is selected + private boolean followRunning; ChaosTab(MonitorContext ctx) { this(ctx, new ChaosControl(ctx)); @@ -125,11 +127,20 @@ class ChaosTab extends AbstractTableTab { toggleExplore(); return true; } - if (explore) { - // the exploration is read-only: the keys of the rules do not act on it + IntegrationInfo info = ctx.findSelectedIntegration(); + if (ke.isChar('L') && control.step() == ChaosControl.Step.NONE) { + // let there be chaos: pick the rules to try, for a budget of time (CAMEL-26760) + control.openExplore(info); + return true; + } + if (explore && control.step() == ChaosControl.Step.NONE) { + // the exploration is read-only: the keys of the rules do not act on it, but one started here can be stopped + if (ke.isChar('x') && info != null && control.explorer().isRunning(info.pid)) { + control.explorer().stop(); + return true; + } return false; } - IntegrationInfo info = ctx.findSelectedIntegration(); ChaosInfo.Rule rule = info != null && info.chaos != null ? selectedRule(info.chaos) : null; if (ke.isKey(KeyCode.F8) && ke.hasShift() && control.step() == ChaosControl.Step.NONE) { // explain with AI what the selected rule did (CAMEL-26702) @@ -144,7 +155,15 @@ class ChaosTab extends AbstractTableTab { ctx.askAiCallback.ask(ChaosAiPrompt.ofTest(info, rule)); return true; } - return control.handleKey(ke, info, rule); + boolean used = control.handleKey(ke, info, rule); + if (control.takeExploreStarted()) { + // the exploration that was just started + if (!explore) { + toggleExplore(); + } + followRunning = true; + } + return used; } @Override @@ -190,6 +209,7 @@ class ChaosTab extends AbstractTableTab { @Override public void navigateUp() { if (!control.isOpen()) { + followRunning = false; super.navigateUp(); } } @@ -197,6 +217,7 @@ class ChaosTab extends AbstractTableTab { @Override public void navigateDown() { if (!control.isOpen()) { + followRunning = false; super.navigateDown(); } } @@ -230,8 +251,14 @@ class ChaosTab extends AbstractTableTab { return; } if (explore) { - lastTableArea = ChaosExploreView.render(frame, area, chaos.exploration, tableState, info.name); + int experiments = chaos.exploration != null ? chaos.exploration.experiments().size() : 0; + if (followRunning && experiments > 0 && control.explorer().isRunning(info.pid)) { + tableState.select(experiments - 1); + } + lastTableArea = ChaosExploreView.render(frame, area, chaos.exploration, tableState, info.name, + control.explorer().progress(info.pid)); lastDetailArea = null; + control.render(frame, area); return; } int tableHeight = Math.max(5, Math.min(chaos.rules.size() + 3, area.height() / 3)); @@ -849,14 +876,26 @@ class ChaosTab extends AbstractTableTab { hint(spans, "Esc", "back"); return; } - if (explore) { + if (explore && control.step() == ChaosControl.Step.NONE) { hint(spans, "↑↓", "experiment"); + if (control.explorer().isRunning(info.pid)) { + hint(spans, "x", "stop"); + } else { + hint(spans, "L", "let there be chaos"); + } hint(spans, "l", "rules"); hint(spans, "Esc", "back"); return; } + if (explore) { + // the keys of the open dialog + control.renderFooter(spans, false); + return; + } control.renderFooter(spans, !info.chaos.rules.isEmpty()); - hint(spans, "l", "explore"); + if (control.step() == ChaosControl.Step.NONE) { + hint(spans, "l", "explore"); + } } @Override diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/DialogHelper.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/DialogHelper.java index 48e0f8ee4d6c..25aaffcfc818 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/DialogHelper.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/DialogHelper.java @@ -168,15 +168,20 @@ final class DialogHelper { return renderConfirm(frame, area, title, message, detail, acceptKey, acceptLabel, extraKey, extraLabel, null); } - /** As above, recording where the keys of the dialog are drawn in {@code clicks} (when not null). */ + /** + * As above, recording where the keys of the dialog are drawn in {@code clicks} (when not null). Without an + * {@code acceptKey} it only waits, such as while the integration is asked: Esc cancels. + */ static Rect renderConfirm( Frame frame, Rect area, String title, String message, String detail, String acceptKey, String acceptLabel, String extraKey, String extraLabel, HintClicks clicks) { Style accent = Theme.warning(); String titleText = " " + title + " "; String msg = message.trim(); - Line hints = extraKey != null - ? TuiHelper.hintLine(acceptKey, acceptLabel, extraKey, extraLabel, "Esc", "cancel") + Line hints = acceptKey == null + ? TuiHelper.hintLine("Esc", "cancel") + : extraKey != null + ? TuiHelper.hintLine(acceptKey, acceptLabel, extraKey, extraLabel, "Esc", "cancel") : TuiHelper.hintLine(acceptKey, acceptLabel, "Esc", "cancel"); int contentW = Math.max(CharWidth.of(msg), Math.max(detail != null ? CharWidth.of(detail) : 0, hints.width())); int popupW = clampWidth(area, CONFIRM_MIN_WIDTH, Math.max(contentW + 6, CharWidth.of(titleText) + 4)); diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/chaos.md b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/chaos.md index d9cfabb1b6ae..8247f935b2e6 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/chaos.md +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/chaos.md @@ -76,6 +76,20 @@ at a dead letter channel, the routes that failed outside its expectations, and why it could not decide (such as faults that never fired). **l** or Esc shows the rules again. +**L** starts an exploration here, by hand: pick the rules to try from the +suggestions of the integration (the same as **w**), the most worth trying +first, each with what should hold despite its faults. The single faults +not tried yet are picked, of routes that get messages by themselves (a +timer, or a route they are called from); a route whose messages come from +outside is marked *needs messages*: send it some while its rule runs, or +its verdict says nothing. Space picks a rule, Left and Right change the +budget (2m to 30m), and Enter starts. The picked rules are tried one at a +time, each for its share of the budget, and each ends with PASS, FAIL or +INCONCLUSIVE; the line above the experiments says which rule is tried and +how long it has left. **x** stops it: the rules are removed at once. Every +rule is removed when the budget ends, also when the TUI is closed, and the +routes are not changed. One exploration runs at a time. + ## Keys - **a** — Add a rule: pick what should go wrong from the known rules of @@ -96,6 +110,10 @@ Esc shows the rules again. was tried in the route and what is not tried yet - **e** — Change the selected rule: the line starts with the rule as it is - **l** — Explore: the exploration and its experiments, or the rules again +- **L** — Let there be chaos: pick the rules to try and the budget, and + they are tried one at a time (see Explore above). In the dialog: Space + picks, Left and Right change the budget, Enter starts +- **x** — On the exploration: stop the exploration started with **L** - **x** — Remove the selected rule - **c** — Clear: remove all rules (the report keeps what they did) - **R** — A new experiment: no rules and a new report, with the same seed diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosExploreTest.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosExploreTest.java new file mode 100644 index 000000000000..474ecb9f617b --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/ChaosExploreTest.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands.tui; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import dev.tamboui.text.Span; +import dev.tamboui.tui.event.KeyCode; +import dev.tamboui.tui.event.KeyEvent; +import dev.tamboui.tui.event.KeyModifiers; +import org.apache.camel.dsl.jbang.core.common.ChaosExperimentRun; +import org.apache.camel.util.json.JsonObject; +import org.apache.camel.util.json.Jsoner; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +/** + * Let there be chaos from the Chaos screen ({@code L}): the rules to pick from the suggestions of the integration, the + * budget, and the exploration that tries the picked rules one at a time. + */ +class ChaosExploreTest { + + private static final String SUGGESTIONS = """ + {"suggestions": [ + {"route": "orders", "kind": "errors", "title": "Returns an error (HTTP status)", + "rule": "endpoint=http://localhost:8080/payment error status=503 rate=0.25 until=10m expect=orders:failed<=0", + "why": "an HTTP call without a circuit breaker", "feed": {"kind": "self", "fed": true}}, + {"route": "orders", "kind": "lost", "title": "The route after a hand-off fails", + "rule": "route=shipping error rate=0.2 until=2m expect=orders->shipping:lost<=0 within=3m", + "why": "a hand-off over seda", "feed": {"kind": "self", "fed": true}}, + {"route": "refunds", "kind": "consumer", "title": "The route fails", + "rule": "route=refunds error rate=0.05 until=10m expect=refunds:failed<=0", + "why": "an input from outside", "feed": {"kind": "send"}}, + {"route": "payment", "kind": "lifecycle", "title": "The route stalls", + "rule": "route=payment stall duration=20s expect=payment:failed<=0", "tried": true, + "why": "stops taking messages", "feed": {"kind": "http", "fed": true}}, + {"route": "orders", "kind": "dead letter channel", "title": "The dead letter channel is unreachable", + "rule": "endpoint=direct:parked error at=dlq expect=orders:failed<=0", + "why": "fails only when another fault sends it a message", "feed": {"kind": "self", "fed": true}}, + {"route": "orders", "kind": "combined", "title": "An outage while the dead letter channel fails", + "rule": "endpoint=http://localhost:8080/payment error status=503 until=1m expect=orders:failed<=0", + "rules": ["endpoint=http://localhost:8080/payment error status=503 until=1m expect=orders:failed<=0", + "endpoint=direct:parked error at=dlq until=1m"], + "why": "two faults at once", "feed": {"kind": "self", "fed": true}} + ]} + """; + + private IntegrationInfo info; + private ChaosTab tab; + private FakeIntegration integration; + private ChaosExploreRun explorer; + + @BeforeEach + void setUp() throws Exception { + Theme.resetForTesting(); + info = new IntegrationInfo(); + info.pid = "1234"; + info.name = "shop"; + info.chaos = ChaosInfo.parse((JsonObject) Jsoner.deserialize(""" + {"enabled": true, "rules": [], "report": {"verdict": "PASS", "final": true, "seed": 42}} + """)); + MonitorContext ctx = new MonitorContext(new AtomicReference<>(List.of(info)), new AtomicReference<>(List.of())); + ctx.selectedPid = "1234"; + integration = new FakeIntegration(); + explorer = new ChaosExploreRun(pid -> integration); + JsonObject suggestions = (JsonObject) Jsoner.deserialize(SUGGESTIONS); + tab = new ChaosTab(ctx, new ChaosControl(ctx, request -> suggestions, () -> null, explorer)); + TuiTestHelper.renderToString(tab, 160, 40); + } + + private static ChaosExploreDialog dialog() throws Exception { + return new ChaosExploreDialog("shop", (JsonObject) Jsoner.deserialize(SUGGESTIONS)); + } + + private void key(char ch) { + tab.handleKeyEvent(KeyEvent.ofChar(ch, KeyModifiers.NONE)); + } + + private void press(KeyCode code) { + tab.handleKeyEvent(KeyEvent.ofKey(code, KeyModifiers.NONE)); + } + + private String footer() { + List<Span> spans = new ArrayList<>(); + tab.renderFooter(spans); + return spans.stream().map(Span::content).reduce("", String::concat); + } + + @Test + void theFaultsNotTriedWhoseRoutesGetMessagesArePicked() throws Exception { + ChaosExploreDialog dialog = dialog(); + assertThat(dialog.items()).hasSize(6); + // not refunds (its messages come from outside), not the stall (tried), not the dead letter channel on its own + // (it fails only when another fault sends it a message), and the pair after the single ones + assertThat(dialog.picked()).extracting(ChaosExploreDialog.Item::title) + .containsExactly("Returns an error (HTTP status)", "The route after a hand-off fails", + "An outage while the dead letter channel fails"); + assertThat(dialog.items().get(0).expect()).isEqualTo("orders:failed<=0"); + assertThat(dialog.items().get(5).rule()).contains("; endpoint=direct:parked error at=dlq"); + // 10 minutes for three rules + assertThat(dialog.budget()).isEqualTo("10m"); + assertThat(dialog.share()).isEqualTo(200_000); + } + + @Test + void spacePicksAndTheArrowsChangeTheBudget() throws Exception { + ChaosExploreDialog dialog = dialog(); + dialog.handleKey(KeyEvent.ofKey(KeyCode.DOWN, KeyModifiers.NONE)); + dialog.handleKey(KeyEvent.ofKey(KeyCode.DOWN, KeyModifiers.NONE)); + dialog.handleKey(KeyEvent.ofChar(' ', KeyModifiers.NONE)); + assertThat(dialog.picked()).hasSize(4); + dialog.handleKey(KeyEvent.ofKey(KeyCode.LEFT, KeyModifiers.NONE)); + dialog.handleKey(KeyEvent.ofKey(KeyCode.LEFT, KeyModifiers.NONE)); + assertThat(dialog.budget()).isEqualTo("2m"); + assertThat(dialog.share()).isEqualTo(30_000); + assertThat(dialog.cannotStart()).isNull(); + + // too many rules for the budget: each needs time for its faults and messages + dialog.handleKey(KeyEvent.ofKey(KeyCode.DOWN, KeyModifiers.NONE)); + dialog.handleKey(KeyEvent.ofChar(' ', KeyModifiers.NONE)); + assertThat(dialog.picked()).hasSize(5); + assertThat(dialog.cannotStart()).startsWith("Too many rules for 2m"); + assertThat(dialog.handleKey(KeyEvent.ofKey(KeyCode.ENTER, KeyModifiers.NONE))) + .isEqualTo(ChaosExploreDialog.Outcome.NONE); + } + + @Test + void eachRuleRunsForItsShare() { + assertThat(ChaosExploreRun.forShare( + "route=shipping error rate=0.2 until=2m expect=orders->shipping:lost<=0 within=3m", 150_000)) + .isEqualTo("route=shipping error rate=0.2 until=150s expect=orders->shipping:lost<=0 within=150s"); + assertThat(ChaosExploreRun.forShare("route=payment stall duration=20s expect=payment:failed<=0", 60_000)) + .isEqualTo("route=payment stall duration=20s expect=payment:failed<=0 until=60s"); + // two faults at once: both end with the share + assertThat(ChaosExploreRun.forShare("endpoint=a error until=1m; endpoint=b error at=dlq until=1m", 30_000)) + .isEqualTo("endpoint=a error until=30s; endpoint=b error at=dlq until=30s"); + } + + @Test + void letThereBeChaosTriesThePickedRulesOneAtATime() { + key('L'); + assertThat(tab.isTextInputActive()).isTrue(); + String screen = TuiTestHelper.renderToString(tab, 160, 40); + assertThat(screen).contains("Let there be chaos: shop", "Budget", "10m", "3 rules, 3m20s each", + "[x] Returns an error (HTTP status)", "[ ] The route fails", "needs messages"); + assertThat(footer()).contains("Space", "pick", "budget", "Enter", "start"); + + press(KeyCode.ENTER); + // the exploration shows + assertThat(tab.isExplore()).isTrue(); + await().atMost(10, TimeUnit.SECONDS).until(() -> !explorer.isRunning("1234")); + assertThat(integration.commands).containsExactly( + "explore-start", "experiment-begin", "experiment-end", "experiment-begin", "experiment-end", + "experiment-begin", "experiment-end", "explore-finish"); + assertThat(integration.budget).isEqualTo("10m"); + // each rule for its share of the time left (the fake ends each experiment at once, so the shares grow) + assertThat(integration.rules).hasSize(3); + assertThat(integration.rules.get(0)).matches( + "endpoint=http://localhost:8080/payment error status=503 rate=0\\.25 until=19\\ds expect=orders:failed<=0"); + assertThat(integration.rules.get(1)).matches( + "route=shipping error rate=0\\.2 until=29\\ds expect=orders->shipping:lost<=0 within=29\\ds"); + // two faults at once: both end with the share + assertThat(integration.rules.get(2)).matches( + "endpoint=http://localhost:8080/payment error status=503 until=59\\ds expect=orders:failed<=0; " + + "endpoint=direct:parked error at=dlq until=59\\ds"); + assertThat(integration.hypotheses).containsExactly( + "orders: Returns an error (HTTP status)", "orders: The route after a hand-off fails", + "orders: An outage while the dead letter channel fails"); + assertThat(explorer.progress("1234")).isEqualTo("Done: 3 rules tried, 3 FAIL"); + assertThat(TuiTestHelper.renderToString(tab, 160, 40)).contains("Done: 3 rules tried, 3 FAIL"); + } + + @Test + void stopEndsTheExploration() { + integration.decided = false; + key('L'); + press(KeyCode.ENTER); + await().atMost(10, TimeUnit.SECONDS).until(() -> integration.commands.contains("experiment-begin")); + assertThat(footer()).contains("x", "stop"); + assertThat(TuiTestHelper.renderToString(tab, 160, 40)).contains("Rule 1 of 3: orders, Returns an error"); + + key('x'); + await().atMost(10, TimeUnit.SECONDS).until(() -> !explorer.isRunning("1234")); + // the experiment is ended and the exploration finished, which removes the rules + assertThat(integration.commands).endsWith("experiment-end", "explore-finish"); + assertThat(explorer.progress("1234")).startsWith("Stopped after 0 rules tried"); + assertThat(footer()).contains("L", "let there be chaos"); + } + + @Test + void oneExplorationAtATime() throws Exception { + info.chaos = ChaosInfo.parse((JsonObject) Jsoner.deserialize(""" + {"enabled": true, "rules": [], + "report": {"verdict": "PASS", "final": false, "seed": 42, + "exploration": {"running": true, "budget": 600000, "timeLeft": 300000, "experiments": []}}} + """)); + key('L'); + assertThat(TuiTestHelper.renderToString(tab, 160, 40)).contains("An exploration runs in shop already"); + } + + /** The integration, as the chaos action and the status answer: every experiment fails at once. */ + private static final class FakeIntegration implements ChaosExperimentRun.Integration { + final List<String> commands = new CopyOnWriteArrayList<>(); + final List<String> rules = new CopyOnWriteArrayList<>(); + final List<String> hypotheses = new CopyOnWriteArrayList<>(); + volatile String budget; + volatile boolean decided = true; + private volatile int number; + + @Override + public JsonObject chaos(String command, Consumer<JsonObject> options) { + JsonObject request = new JsonObject(); + options.accept(request); + commands.add(command); + JsonObject answer = new JsonObject(); + switch (command) { + case "explore-start" -> budget = request.getString("budget"); + case "experiment-begin" -> { + number++; + rules.add(request.getString("rule")); + hypotheses.add(request.getString("hypothesis")); + answer.put("number", number); + } + case "experiment-end" -> { + answer.put("number", number); + answer.put("verdict", "FAIL"); + answer.put("open", false); + } + default -> { + // finished + } + } + return answer; + } + + @Override + public JsonObject send(ChaosExperimentRun.Feed feed) { + throw new UnsupportedOperationException(); + } + + @Override + public JsonObject status() { + JsonObject experiment = new JsonObject(); + experiment.put("number", number); + experiment.put("verdict", decided ? "FAIL" : "PENDING"); + experiment.put("open", !decided); + JsonObject exploration = new JsonObject(); + exploration.put("experiments", List.of(experiment)); + exploration.put("timeLeft", 60_000); + JsonObject report = new JsonObject(); + report.put("exploration", exploration); + JsonObject chaos = new JsonObject(); + chaos.put("report", report); + JsonObject status = new JsonObject(); + status.put("chaos", chaos); + return status; + } + } +}
