This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch CAMEL-26691-timeline-tui in repository https://gitbox.apache.org/repos/asf/camel.git
commit a1c8272d36a62677edc50bc69c9053efc2c37656 Author: Claus Ibsen <[email protected]> AuthorDate: Fri Nov 6 23:20:13 2026 +0100 CAMEL-26691: camel-jbang - a timeline of an integration: errors and route outcomes per second, with chaos rules as a layer Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../core/commands/tui/DataRefreshService.java | 6 +- .../jbang/core/commands/tui/MonitorContext.java | 2 + .../dsl/jbang/core/commands/tui/TabRegistry.java | 3 + .../dsl/jbang/core/commands/tui/Timeline.java | 415 ++++++++++++++ .../dsl/jbang/core/commands/tui/TimelineTab.java | 636 +++++++++++++++++++++ .../dsl/jbang/core/commands/tui/TuiIcons.java | 1 + .../src/main/resources/tui/help/timeline.md | 65 +++ .../core/commands/tui/TimelineTabRenderTest.java | 181 ++++++ .../dsl/jbang/core/commands/tui/TimelineTest.java | 162 ++++++ 9 files changed, 1470 insertions(+), 1 deletion(-) diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/DataRefreshService.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/DataRefreshService.java index 11b24aa75ba8..dac93f316267 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/DataRefreshService.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/DataRefreshService.java @@ -323,6 +323,7 @@ class DataRefreshService { IntegrationInfo info = StatusParser.parseIntegration(ph, root); if (info != null && ctx != null) { ctx.chaosMarks.record(info); + ctx.timeline.record(info, System.currentTimeMillis()); } if (info != null) { IntegrationInfo prev = previousByPid.get(info.pid); @@ -343,7 +344,9 @@ class DataRefreshService { } } if (ctx != null) { - ctx.chaosMarks.retain(pids.stream().map(String::valueOf).collect(Collectors.toSet())); + Set<String> live = pids.stream().map(String::valueOf).collect(Collectors.toSet()); + ctx.chaosMarks.retain(live); + ctx.timeline.retain(live); } if (!fullScan && ctx.selectedPid != null && !overviewTab) { boolean checkLiveness = now - lastLivenessCheckTime >= LIVENESS_CHECK_INTERVAL_MS; @@ -758,6 +761,7 @@ class DataRefreshService { if (root != null) { List<ErrorInfo> parsed = StatusParser.parseErrors(root); sel.errors = List.copyOf(parsed); + ctx.timeline.recordErrors(sel.pid, sel.errors, System.currentTimeMillis()); } } catch (Exception e) { // ignore diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/MonitorContext.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/MonitorContext.java index b207871f1fd7..4de39d756175 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/MonitorContext.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/MonitorContext.java @@ -96,6 +96,8 @@ class MonitorContext { Consumer<ChaosHere.Target> chaosHere; // the exchanges chaos rules faulted, for the marks on the Activity, Errors and Inspect tabs final ChaosMarks chaosMarks = new ChaosMarks(); + // what happened second by second, for the Timeline screen (CAMEL-26691) + final Timeline timeline = new Timeline(); BiConsumer<String, String> openMarkdownCallback; /** Opens markdown in the doc viewer scrolled to a heading: title, markdown, heading (may be null). */ MarkdownOpener openMarkdownAtCallback; diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TabRegistry.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TabRegistry.java index e8bf01021d52..78bf455f19f7 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TabRegistry.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TabRegistry.java @@ -74,6 +74,7 @@ class TabRegistry { private HistoryTab historyTab; private CircuitBreakerTab circuitBreakerTab; private ChaosTab chaosTab; + private TimelineTab timelineTab; private ErrorsTab errorsTab; private ActivityTab activityTab; private MetricsTab metricsTab; @@ -143,6 +144,7 @@ class TabRegistry { historyTab = new HistoryTab(ctx, dataService.traces(), dataService.traceFilePositions()); circuitBreakerTab = new CircuitBreakerTab(ctx, dataService.metrics()); chaosTab = new ChaosTab(ctx); + timelineTab = new TimelineTab(ctx); errorsTab = new ErrorsTab(ctx); activityTab = new ActivityTab(ctx); metricsTab = new MetricsTab(ctx); @@ -189,6 +191,7 @@ class TabRegistry { "Routing", List.of(), info -> "SupervisingRouteController".equals(info.routeControllerType)), // Observability + new MoreTab(TuiIcons.TAB_TIMELINE, "Timeline", "Time&line", timelineTab, "Observability"), new MoreTab( TuiIcons.TAB_CHAOS, "Chaos", "C&haos", chaosTab, "Observability", List.of("chaos")), diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/Timeline.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/Timeline.java new file mode 100644 index 000000000000..6b1f4c5e65d9 --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/Timeline.java @@ -0,0 +1,415 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands.tui; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * What happened to an integration second by second, for the timeline screen (CAMEL-26691): the counters of each route + * at each second, from which the exchanges per second that completed, were recovered by redelivery, were handled by the + * error handler or failed are worked out; the errors per second by kind; and with camel-chaos, the faults of each rule + * per second and the seconds a rule had a route down. It is recorded at every refresh of an integration, so the screen + * opens on the last minutes, not on an empty chart. + */ +final class Timeline { + + /** The seconds kept. */ + static final int SECONDS = 300; + // a gap between two samples of at most this many seconds (a slow refresh) is spread over its seconds + private static final int SPREAD = 10; + + private static final int TOTAL = 0; + private static final int FAILED = 1; + private static final int HANDLED = 2; + private static final int REDELIVERIES = 3; + private static final int INFLIGHT = 4; + private static final int MEAN = 5; + + private static final Pattern STATUS_CODE = Pattern.compile("statusCode: ?(\\d{3})"); + + private final Map<String, Recording> byPid = new ConcurrentHashMap<>(); + + /** The exchanges of a route per second, oldest first; a second before the first sample is -1. */ + record RouteSeries(String route, long[] completed, long[] recovered, long[] handled, long[] failed, long[] inflight, + long[] mean, long[] down) { + + long total(long[] values) { + long sum = 0; + for (long v : values) { + sum += Math.max(0, v); + } + return sum; + } + + /** Whether nothing happened in the window: no exchange, none in flight, and the route never down. */ + boolean quiet() { + return total(completed) + total(recovered) + total(handled) + total(failed) + total(inflight) + + total(down) == 0; + } + } + + /** One kind of cause per second, oldest first: an error kind, or the faults of a chaos rule. */ + record CauseSeries(String label, String rule, long[] values) { + } + + private static final class Recording { + // the counters of each route at the end of a second + final TreeMap<Long, Map<String, long[]>> routes = new TreeMap<>(); + // the new errors per second, by their label + final TreeMap<Long, Map<String, Long>> errors = new TreeMap<>(); + // the faults per second of each chaos rule, and the routes a rule had down + final TreeMap<Long, Map<String, Long>> faults = new TreeMap<>(); + final TreeMap<Long, Set<String>> down = new TreeMap<>(); + final Map<String, String> ruleLabels = new LinkedHashMap<>(); + // the highest repeat count seen of each kind of error, and the newest time of its entries + final Map<String, Long> repeats = new HashMap<>(); + final Map<String, Long> newest = new HashMap<>(); + boolean errorsSeen; + + void trim(long second) { + long oldest = second - SECONDS; + routes.headMap(oldest).clear(); + errors.headMap(oldest).clear(); + faults.headMap(oldest).clear(); + down.headMap(oldest).clear(); + } + } + + /** Records the route counters and the chaos rules of an integration at a refresh. */ + void record(IntegrationInfo info, long now) { + if (info == null || info.pid == null) { + return; + } + long second = now / 1000; + Recording rec = byPid.computeIfAbsent(info.pid, k -> new Recording()); + synchronized (rec) { + Map<String, long[]> counters = new LinkedHashMap<>(); + for (RouteInfo r : info.routes) { + if (r.routeId != null) { + counters.put(r.routeId, + new long[] { r.total, r.failed, r.failuresHandled, r.redeliveries, r.inflight, r.meanTime }); + } + } + rec.routes.put(second, counters); + if (info.chaos != null) { + recordChaos(rec, info.chaos, second); + } + rec.trim(second); + } + } + + private static void recordChaos(Recording rec, ChaosInfo chaos, long second) { + Set<String> down = new LinkedHashSet<>(); + for (ChaosInfo.Rule r : chaos.rules) { + if (r.id == null) { + continue; + } + rec.ruleLabels.put(r.id, ruleLabel(r)); + // the last value is the current second of the integration + long[] perSecond = r.faultsPerSecond; + for (int k = 0; k < perSecond.length; k++) { + long s = second - (perSecond.length - 1) + k; + Map<String, Long> at = rec.faults.computeIfAbsent(s, x -> new LinkedHashMap<>()); + if (perSecond[k] > 0) { + at.put(r.id, perSecond[k]); + } else { + at.remove(r.id); + } + } + if ("down".equals(r.lifecycleState) && r.lifecycleRoute != null) { + down.add(r.lifecycleRoute); + } + } + if (!down.isEmpty()) { + rec.down.put(second, down); + } else { + rec.down.remove(second); + } + } + + /** A short label of a chaos rule for the legend: r1 error 503 (stock). */ + static String ruleLabel(ChaosInfo.Rule r) { + String fault = r.faultShort(); + String target = ChaosTab.target(r); + int eq = target.indexOf('='); + String where = eq > 0 ? target.substring(eq + 1) : target; + int colon = where.lastIndexOf(':'); + if (colon >= 0 && colon < where.length() - 1) { + where = where.substring(colon + 1); + } + where = where.replace("*", "").replaceAll("^/+", ""); + int slash = where.lastIndexOf('/'); + if (slash >= 0 && slash < where.length() - 1) { + where = where.substring(slash + 1); + } + return r.id + " " + fault + (where.isEmpty() ? "" : " (" + where + ")"); + } + + /** + * Records the errors of an integration (the error registry, as the Errors tab reads it). An entry is one exchange + * that failed, and its repeat count says how often its kind (route, node and exception) failed so far, so the new + * failures of a kind are the rise of its count. The first time, the entries are counted at their own seconds. An + * error a chaos rule made is left out: the faults of the rule are a layer of their own. + */ + void recordErrors(String pid, List<ErrorInfo> errors, long now) { + if (pid == null || errors == null) { + return; + } + Recording rec = byPid.computeIfAbsent(pid, k -> new Recording()); + synchronized (rec) { + Map<String, List<ErrorInfo>> byKind = new LinkedHashMap<>(); + for (ErrorInfo e : errors) { + if (e.properties.get(ChaosMarks.PROPERTY) != null) { + continue; + } + byKind.computeIfAbsent(e.routeId + "|" + e.nodeId + "|" + e.exceptionType, k -> new ArrayList<>()).add(e); + } + for (Map.Entry<String, List<ErrorInfo>> en : byKind.entrySet()) { + recordKind(rec, en.getKey(), en.getValue(), now); + } + rec.errorsSeen = true; + rec.trim(now / 1000); + } + } + + private static void recordKind(Recording rec, String kind, List<ErrorInfo> entries, long now) { + long count = 0; + long newestTs = 0; + for (ErrorInfo e : entries) { + count = Math.max(count, e.repeatCount); + newestTs = Math.max(newestTs, e.timestamp); + } + String label = errorLabel(entries.get(0)); + Long before = rec.repeats.get(kind); + Long seenTs = rec.newest.get(kind); + rec.repeats.put(kind, count); + rec.newest.put(kind, newestTs); + long oldest = now - SECONDS * 1000L; + if (!rec.errorsSeen || before == null) { + // what was there already: each entry kept at its own second, and the failures of the kind that are no + // longer kept spread from its first failure to the oldest entry kept + long oldestKept = Long.MAX_VALUE; + long first = 0; + for (ErrorInfo e : entries) { + oldestKept = Math.min(oldestKept, e.timestamp); + first = first == 0 ? e.repeatFirstTimestamp : Math.min(first, e.repeatFirstTimestamp); + if (e.timestamp > oldest) { + addError(rec, e.timestamp / 1000, label, 1); + } + } + long rest = count - entries.size(); + if (rest > 0 && first > 0 && first <= oldestKept) { + long from = Math.max(first, oldest) / 1000; + long to = oldestKept / 1000; + long span = to - from + 1; + // only the part of the spread inside the seconds kept + long inside = first >= oldest ? rest : rest * span / Math.max(1, to - first / 1000 + 1); + for (long t = from; t <= to; t++) { + long n = share(inside, span, t - from); + if (n > 0) { + addError(rec, t, label, n); + } + } + } + return; + } + long rise = count >= before ? count - before : count; + if (rise <= 0) { + return; + } + // the new entries at their own seconds, and the rest of the rise (entries no longer kept) at the newest + for (ErrorInfo e : entries) { + if (rise > 1 && e.timestamp > oldest && (seenTs == null || e.timestamp > seenTs) + && e.timestamp != newestTs) { + addError(rec, e.timestamp / 1000, label, 1); + rise--; + } + } + addError(rec, (newestTs > 0 ? newestTs : now) / 1000, label, rise); + } + + private static void addError(Recording rec, long second, String label, long n) { + rec.errors.computeIfAbsent(second, s -> new LinkedHashMap<>()).merge(label, n, Long::sum); + } + + /** The label of an error in the legend: the simple name of the exception, and the HTTP status if it has one. */ + static String errorLabel(ErrorInfo e) { + String type = e.exceptionType != null ? e.exceptionType : "Error"; + int dot = type.lastIndexOf('.'); + String label = dot >= 0 ? type.substring(dot + 1) : type; + if (e.exceptionMessage != null) { + Matcher m = STATUS_CODE.matcher(e.exceptionMessage); + if (m.find()) { + label += " " + m.group(1); + } + } + return label; + } + + /** The routes in the window that ends at the second, in the order of the integration. */ + List<RouteSeries> routes(String pid, List<String> order, long endSecond, int seconds) { + Recording rec = pid != null ? byPid.get(pid) : null; + if (rec == null) { + return List.of(); + } + synchronized (rec) { + Set<String> names = new LinkedHashSet<>(order); + rec.routes.values().forEach(m -> names.addAll(m.keySet())); + List<RouteSeries> answer = new ArrayList<>(); + for (String name : names) { + answer.add(series(rec, name, endSecond, seconds)); + } + return answer; + } + } + + private static RouteSeries series(Recording rec, String route, long endSecond, int seconds) { + long start = endSecond - seconds + 1; + long[] completed = filled(seconds); + long[] recovered = filled(seconds); + long[] handled = filled(seconds); + long[] failed = filled(seconds); + long[] inflight = filled(seconds); + long[] mean = filled(seconds); + long[] down = new long[seconds]; + long prevSecond = -1; + long[] prev = null; + for (Map.Entry<Long, Map<String, long[]>> en : rec.routes.headMap(endSecond, true).entrySet()) { + long[] cur = en.getValue().get(route); + if (cur == null) { + continue; + } + long s = en.getKey(); + if (s >= start) { + inflight[(int) (s - start)] = cur[INFLIGHT]; + } + if (prev != null && s >= start) { + long dTotal = Math.max(0, cur[TOTAL] - prev[TOTAL]); + long dFailed = Math.max(0, cur[FAILED] - prev[FAILED]); + long dHandled = Math.max(0, cur[HANDLED] - prev[HANDLED]); + long dRecovered = Math.max(0, cur[REDELIVERIES] - prev[REDELIVERIES]); + long dOk = Math.max(0, dTotal - dFailed - dHandled - dRecovered); + // the mean of the exchanges done since the last sample, from the means of all of them + long dTime = cur[MEAN] * cur[TOTAL] - prev[MEAN] * prev[TOTAL]; + long m = dTotal > 0 ? Math.max(0, dTime / dTotal) : -1; + long gap = s - prevSecond; + long from = gap <= SPREAD ? prevSecond + 1 : s; + long span = s - from + 1; + for (long t = Math.max(from, start); t <= s; t++) { + int i = (int) (t - start); + // the later seconds of the gap take the remainder + long k = t - from; + completed[i] = share(dOk, span, k); + recovered[i] = share(dRecovered, span, k); + handled[i] = share(dHandled, span, k); + failed[i] = share(dFailed, span, k); + mean[i] = m; + if (t < s && inflight[i] < 0) { + inflight[i] = cur[INFLIGHT]; + } + } + } + prev = cur; + prevSecond = s; + } + for (Map.Entry<Long, Set<String>> en : rec.down.subMap(start, true, endSecond, true).entrySet()) { + if (en.getValue().contains(route)) { + down[(int) (en.getKey() - start)] = 1; + } + } + return new RouteSeries(route, completed, recovered, handled, failed, inflight, mean, down); + } + + /** The k-th of n parts of a count: the parts add up to the count, the later ones a little bigger. */ + private static long share(long count, long n, long k) { + long base = count / n; + long rest = count % n; + return base + (k >= n - rest ? 1 : 0); + } + + private static long[] filled(int seconds) { + long[] a = new long[seconds]; + Arrays.fill(a, -1); + return a; + } + + /** + * The causes in the window: the errors per second by label, then the faults of each chaos rule. Each series is + * oldest first. The errors are known only from the first time the Errors tab or the timeline read them. + */ + List<CauseSeries> causes(String pid, long endSecond, int seconds) { + Recording rec = pid != null ? byPid.get(pid) : null; + if (rec == null) { + return List.of(); + } + long start = endSecond - seconds + 1; + synchronized (rec) { + Map<String, long[]> errors = new LinkedHashMap<>(); + for (Map.Entry<Long, Map<String, Long>> en : rec.errors.subMap(start, true, endSecond, true).entrySet()) { + int i = (int) (en.getKey() - start); + en.getValue().forEach((label, n) -> errors.computeIfAbsent(label, l -> new long[seconds])[i] += n); + } + Map<String, long[]> faults = new LinkedHashMap<>(); + for (Map.Entry<Long, Map<String, Long>> en : rec.faults.subMap(start, true, endSecond, true).entrySet()) { + int i = (int) (en.getKey() - start); + en.getValue().forEach((rule, n) -> faults.computeIfAbsent(rule, r -> new long[seconds])[i] += n); + } + List<CauseSeries> answer = new ArrayList<>(); + errors.forEach((label, values) -> answer.add(new CauseSeries(label, null, values))); + faults.forEach((rule, values) -> answer + .add(new CauseSeries(rec.ruleLabels.getOrDefault(rule, rule), rule, values))); + return answer; + } + } + + /** Whether the errors of the integration were read at least once, so an empty cause band means none. */ + boolean errorsSeen(String pid) { + Recording rec = pid != null ? byPid.get(pid) : null; + if (rec == null) { + return false; + } + synchronized (rec) { + return rec.errorsSeen; + } + } + + /** The seconds recorded of the integration, the oldest first: how far back the screen can go. */ + long firstSecond(String pid) { + Recording rec = pid != null ? byPid.get(pid) : null; + if (rec == null) { + return -1; + } + synchronized (rec) { + return rec.routes.isEmpty() ? -1 : rec.routes.firstKey(); + } + } + + void retain(Set<String> pids) { + byPid.keySet().retainAll(pids); + } +} diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTab.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTab.java new file mode 100644 index 000000000000..90b232778894 --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTab.java @@ -0,0 +1,636 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands.tui; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.function.LongSupplier; + +import dev.tamboui.layout.Constraint; +import dev.tamboui.layout.Layout; +import dev.tamboui.layout.Rect; +import dev.tamboui.style.Style; +import dev.tamboui.terminal.Frame; +import dev.tamboui.text.Line; +import dev.tamboui.text.Span; +import dev.tamboui.text.Text; +import dev.tamboui.tui.event.KeyEvent; +import dev.tamboui.tui.event.MouseEvent; +import dev.tamboui.widgets.block.Block; +import dev.tamboui.widgets.block.BorderType; +import dev.tamboui.widgets.block.Borders; +import dev.tamboui.widgets.paragraph.Paragraph; +import org.apache.camel.util.json.JsonArray; +import org.apache.camel.util.json.JsonObject; + +import static org.apache.camel.dsl.jbang.core.commands.tui.TuiHelper.hint; + +/** + * The timeline of an integration (CAMEL-26691): the last minute on one time axis, one column per second. On top the + * cause, the errors per second by kind, and with camel-chaos the faults of each rule marked π². Under it the effect, a + * line per route of the exchanges per second, coloured by the worst that happened to them that second (failed, handled + * by the error handler, recovered by redelivery, completed), with the seconds a chaos rule had the route down shaded. + * The selected route is shown in detail at the bottom: its outcomes stacked per second, its mean processing time and + * its exchanges in flight. A quiet integration is a calm screen. + */ +class TimelineTab extends AbstractTableTab { + + static final int[] WINDOWS = { 60, 120, 300 }; + private static final String BLOCKS = "βββββ βββ"; + private static final int CAUSE_ROWS = 4; + private static final int DETAIL_ROWS = 4; + private static final int SUFFIX = 24; + private static final int MAX_LABEL = 18; + + private final LongSupplier clock; + private int window; + // the second the screen stays at while paused, or -1 to follow now + private long pausedAt = -1; + private int routeLinesY = -1; + + TimelineTab(MonitorContext ctx) { + this(ctx, System::currentTimeMillis); + } + + TimelineTab(MonitorContext ctx, LongSupplier clock) { + super(ctx); + this.clock = clock; + } + + @Override + protected boolean selectsFirstRow() { + return true; + } + + @Override + protected int getRowCount() { + return routes(ctx.findSelectedIntegration()).size(); + } + + @Override + public void onTabSelected() { + refreshWhileVisible(); + } + + @Override + public void refreshWhileVisible() { + // the errors are read while the screen shows (the refresh records them in the timeline) + DataRefreshService data = ctx.dataService; + List<Long> pids = data != null ? data.selectedPidAsList() : List.of(); + if (!pids.isEmpty()) { + data.refreshErrorData(pids); + } + } + + @Override + protected boolean handleTabKeyEvent(KeyEvent ke) { + if (ke.isChar('w')) { + window = (window + 1) % WINDOWS.length; + return true; + } + if (ke.isChar(' ')) { + pausedAt = pausedAt < 0 ? clock.getAsLong() / 1000 : -1; + return true; + } + if (pausedAt >= 0 && (ke.isLeft() || ke.isRight())) { + IntegrationInfo info = ctx.findSelectedIntegration(); + long first = info != null ? ctx.timeline.firstSecond(info.pid) : -1; + long now = clock.getAsLong() / 1000; + long step = Math.max(5, seconds() / 4); + long to = pausedAt + (ke.isLeft() ? -step : step); + long earliest = first >= 0 ? Math.min(now, first + seconds() - 1) : now; + pausedAt = Math.max(earliest, Math.min(now, to)); + return true; + } + return false; + } + + @Override + public boolean handleEscape() { + if (pausedAt >= 0) { + pausedAt = -1; + return true; + } + return false; + } + + @Override + public boolean handleMouseEvent(MouseEvent me, Rect area) { + if (routeLinesY < 0 || lastTableArea == null) { + return false; + } + return handleTableClick(me, lastTableArea, tableState, getRowCount()); + } + + int seconds() { + return WINDOWS[window]; + } + + boolean isPaused() { + return pausedAt >= 0; + } + + private long endSecond() { + return pausedAt >= 0 ? pausedAt : clock.getAsLong() / 1000; + } + + private List<Timeline.RouteSeries> routes(IntegrationInfo info) { + if (info == null) { + return List.of(); + } + List<String> order = info.routes.stream().map(r -> r.routeId).filter(id -> id != null).toList(); + return ctx.timeline.routes(info.pid, order, endSecond(), seconds()); + } + + @Override + protected void renderContent(Frame frame, Rect area, IntegrationInfo info) { + List<Timeline.RouteSeries> routes = routes(info); + List<Timeline.CauseSeries> causes = ctx.timeline.causes(info.pid, endSecond(), seconds()); + boolean anyCause = causes.stream().anyMatch(c -> sum(c.values()) > 0); + int causeHeight = anyCause ? CAUSE_ROWS + 1 : 1; + // the cause, the legend of the routes, a line per route and the time axis, in a border + int detailHeight = DETAIL_ROWS + 5; + int topHeight = Math.max(6, Math.min(area.height() - detailHeight, causeHeight + routes.size() + 4)); + List<Rect> rows = Layout.vertical() + .constraints(Constraint.length(topHeight), Constraint.length(detailHeight), Constraint.fill()) + .split(area); + int inner = Math.max(20, area.width() - 2); + int label = Math.min(MAX_LABEL, + Math.max(10, routes.stream().mapToInt(r -> r.route().length()).max().orElse(0))) + 3; + int columns = Math.max(10, Math.min(seconds(), inner - label - SUFFIX - 1)); + int per = (int) Math.ceil((double) seconds() / columns); + columns = (seconds() + per - 1) / per; + + List<Line> lines = new ArrayList<>(); + renderCauses(lines, causes, anyCause, info, columns, per, label); + lines.add(Line.from(Span.raw(pad("", label)), Span.styled("Routes ", Theme.label()), + Span.styled("β", Theme.success()), Span.raw(" completed "), + Span.styled("β", Theme.warning()), Span.raw(" recovered "), + Span.styled("β", handledStyle()), Span.raw(" handled "), + Span.styled("β", Theme.error()), Span.raw(" failed "), + Span.styled("β", Theme.error()), Span.raw(" down"))); + Integer sel = tableState.selected(); + routeLinesY = lines.size(); + for (int i = 0; i < routes.size(); i++) { + lines.add(routeLine(routes.get(i), sel != null && sel == i, columns, per, label)); + } + if (routes.isEmpty()) { + lines.add(Line.from(Span.styled(pad("", label) + "no routes yet", Theme.muted()))); + } + lines.add(axis(columns, per, label)); + Rect top = rows.get(0); + // the rows of the routes, as a table with a header line of two would have them, for the clicks + lastTableArea = new Rect(top.x(), top.y() + 1 + routeLinesY - 2, top.width(), routes.size() + 2); + frame.renderWidget(Paragraph.builder().text(Text.from(lines)).block(block(title(info))).build(), top); + + Timeline.RouteSeries selected = sel != null && sel >= 0 && sel < routes.size() ? routes.get(sel) : null; + renderDetail(frame, rows.get(1), selected, columns, per, label); + + } + + private String title(IntegrationInfo info) { + String span = seconds() >= 120 ? (seconds() / 60) + "m" : seconds() + "s"; + StringBuilder sb = new StringBuilder(" Timeline β ").append(info.name != null ? info.name : info.pid) + .append(" β last ").append(span).append(' '); + if (pausedAt >= 0) { + long ago = clock.getAsLong() / 1000 - pausedAt; + sb.append("β paused").append(ago > 0 ? ", " + ChaosTab.shortTime(ago * 1000) + " ago " : " "); + } + return sb.toString(); + } + + // ---- the cause ---- + + private void renderCauses( + List<Line> lines, List<Timeline.CauseSeries> causes, boolean anyCause, IntegrationInfo info, int columns, + int per, int label) { + List<Span> legend = new ArrayList<>(); + legend.add(Span.raw(pad("", label))); + legend.add(Span.styled("Errors/s ", Theme.label())); + List<long[]> values = new ArrayList<>(); + List<Style> styles = new ArrayList<>(); + int colour = 0; + for (Timeline.CauseSeries c : causes) { + if (sum(c.values()) == 0) { + continue; + } + Style style = causeStyle(colour++); + values.add(columnsOf(c.values(), per, false)); + styles.add(style); + legend.add(Span.styled("β ", style)); + legend.add(Span.raw((c.rule() != null ? "π² " : "") + c.label() + " ")); + } + if (!anyCause) { + boolean seen = ctx.timeline.errorsSeen(info.pid); + legend.add(Span.styled(seen ? "none" : "reading the errorsβ¦", Theme.muted())); + if (info.chaos != null && info.chaos.enabled) { + legend.add(Span.styled(" Β· π² no chaos fault", Theme.muted())); + } + lines.add(Line.from(legend)); + return; + } + lines.add(Line.from(legend)); + lines.addAll(stacked(values, styles, CAUSE_ROWS, columns, label)); + } + + private static Style causeStyle(int i) { + Style[] palette = { + Theme.error(), Style.EMPTY.fg(Theme.ACCENT), Theme.notice(), Theme.info(), Theme.warning(), + Style.EMPTY.fg(Theme.diagramTo()) }; + return palette[i % palette.length]; + } + + // ---- the routes ---- + + private Line routeLine(Timeline.RouteSeries r, boolean selected, int columns, int per, int label) { + List<Span> spans = new ArrayList<>(); + String name = r.route().length() > label - 3 ? r.route().substring(0, label - 4) + "β¦" : r.route(); + spans.add(selected + ? Span.styled(pad(">> " + name, label), Theme.selectionBg()) + : Span.raw(pad(" " + name, label))); + long[] completed = columnsOf(r.completed(), per, false); + long[] recovered = columnsOf(r.recovered(), per, false); + long[] handled = columnsOf(r.handled(), per, false); + long[] failed = columnsOf(r.failed(), per, false); + long[] down = columnsOf(r.down(), per, true); + long max = 1; + for (int c = 0; c < columns; c++) { + max = Math.max(max, total(completed[c], recovered[c], handled[c], failed[c])); + } + for (int c = 0; c < columns; c++) { + long t = total(completed[c], recovered[c], handled[c], failed[c]); + if (down[c] > 0) { + spans.add(Span.styled("β", Theme.error())); + } else if (completed[c] < 0) { + spans.add(Span.raw(" ")); + } else if (t == 0) { + spans.add(Span.styled("Β·", Theme.muted())); + } else { + int level = (int) Math.max(1, Math.min(8, Math.ceil(t * 8.0 / max))); + Style style = failed[c] > 0 ? Theme.error() + : handled[c] > 0 ? handledStyle() + : recovered[c] > 0 ? Theme.warning() : Theme.success(); + spans.add(Span.styled(String.valueOf(BLOCKS.charAt(level - 1)), style)); + } + } + spans.add(Span.raw(" ")); + spans.addAll(summary(r)); + return Line.from(spans); + } + + /** The rate of the route over the window, and the worst that happened to its exchanges. */ + private List<Span> summary(Timeline.RouteSeries r) { + List<Span> spans = new ArrayList<>(); + long done = r.total(r.completed()) + r.total(r.recovered()) + r.total(r.handled()) + r.total(r.failed()); + int known = 0; + for (long v : r.completed()) { + if (v >= 0) { + known++; + } + } + if (known == 0) { + spans.add(Span.styled("no data yet", Theme.muted())); + return spans; + } + if (r.total(r.down()) > 0) { + spans.add(Span.styled("down " + r.total(r.down()) + "s ", Theme.error())); + } + spans.add(Span.raw(rate(done, known))); + if (r.total(r.failed()) > 0) { + spans.add(Span.styled(" " + r.total(r.failed()) + " failed", Theme.error())); + } else if (r.total(r.handled()) > 0) { + spans.add(Span.styled(" " + r.total(r.handled()) + " handled", handledStyle())); + } else if (r.total(r.recovered()) > 0) { + spans.add(Span.styled(" " + r.total(r.recovered()) + " recovered", Theme.warning())); + } else if (done == 0) { + spans.add(Span.styled(" idle", Theme.muted())); + } + return spans; + } + + static String rate(long count, int seconds) { + double perSecond = seconds > 0 ? (double) count / seconds : 0; + if (count == 0) { + return "0/s"; + } + return perSecond >= 10 ? String.format(Locale.US, "%.0f/s", perSecond) + : perSecond >= 0.1 ? String.format(Locale.US, "%.1f/s", perSecond) : count + " in " + seconds + "s"; + } + + private Line axis(int columns, int per, int label) { + char[] axis = new char[columns]; + Arrays.fill(axis, ' '); + String start = "-" + axisTime(seconds()); + String middle = "-" + axisTime(seconds() / 2); + String end = "now"; + put(axis, 0, start); + put(axis, columns / 2 - middle.length() / 2, middle); + put(axis, columns - end.length(), pausedAt >= 0 ? "" : end); + if (pausedAt >= 0) { + String at = "-" + ChaosTab.shortTime(Math.max(0, clock.getAsLong() / 1000 - pausedAt) * 1000); + put(axis, columns - at.length(), at); + } + String text = new String(axis); + if (per > 1) { + text += " " + per + "s a column"; + } + return Line.from(Span.raw(pad("", label)), Span.styled(text, Theme.muted())); + } + + /** A time on the axis: 60s, 30s, 2m, 2m30s. */ + static String axisTime(int seconds) { + return seconds <= 90 ? seconds + "s" : ChaosTab.shortTime(seconds * 1000L); + } + + private static void put(char[] axis, int at, String s) { + for (int i = 0; i < s.length() && at + i < axis.length; i++) { + if (at + i >= 0) { + axis[at + i] = s.charAt(i); + } + } + } + + // ---- the selected route ---- + + private void renderDetail(Frame frame, Rect area, Timeline.RouteSeries r, int columns, int per, int label) { + List<Line> lines = new ArrayList<>(); + if (r == null) { + frame.renderWidget(Paragraph.builder().text(Text.from(Line.from(Span.raw("")))) + .block(block(" Route ")).build(), area); + return; + } + List<long[]> values = List.of(columnsOf(r.completed(), per, false), columnsOf(r.recovered(), per, false), + columnsOf(r.handled(), per, false), columnsOf(r.failed(), per, false)); + List<Style> styles = List.of(Theme.success(), Theme.warning(), handledStyle(), Theme.error()); + int rows = Math.max(2, Math.min(DETAIL_ROWS, area.height() - 5)); + lines.addAll(stacked(values, styles, rows, columns, label)); + + long[] mean = columnsOf(r.mean(), per, true); + long maxMean = 0; + long sumMean = 0; + int n = 0; + for (long m : r.mean()) { + if (m >= 0) { + maxMean = Math.max(maxMean, m); + sumMean += m; + n++; + } + } + List<Span> meanLine = new ArrayList<>(); + meanLine.add(Span.styled(pad(" mean time", label), Theme.label())); + // relative to twice the average, so a steady time is a calm line and a slow second stands out + long scale = n > 0 ? Math.max(maxMean, 2 * sumMean / n) : 0; + meanLine.add(Span.styled(spark(mean, scale), Theme.info())); + meanLine.add(Span.raw(" " + (n > 0 + ? "avg " + ChaosTab.shortTime(sumMean / n) + ", max " + + ChaosTab.shortTime(maxMean) + : "no exchange"))); + lines.add(Line.from(meanLine)); + + long[] inflight = columnsOf(r.inflight(), per, true); + long maxInflight = 0; + long last = 0; + for (long v : r.inflight()) { + if (v >= 0) { + maxInflight = Math.max(maxInflight, v); + last = v; + } + } + lines.add(Line.from(Span.styled(pad(" in flight", label), Theme.label()), + Span.styled(spark(inflight, maxInflight), Theme.notice()), + Span.raw(" now " + last + (maxInflight > last ? ", max " + maxInflight : "")))); + + List<Span> totals = new ArrayList<>(); + totals.add(Span.raw(pad("", label))); + addTotal(totals, r.total(r.completed()), "completed", Theme.success()); + addTotal(totals, r.total(r.recovered()), "recovered by redelivery", Theme.warning()); + addTotal(totals, r.total(r.handled()), "handled by the error handler", handledStyle()); + addTotal(totals, r.total(r.failed()), "failed", Theme.error()); + lines.add(Line.from(totals)); + String title = " " + r.route() + " β per second "; + frame.renderWidget(Paragraph.builder().text(Text.from(lines)).block(block(title)).build(), area); + } + + private static void addTotal(List<Span> spans, long count, String words, Style style) { + spans.add(Span.styled(String.valueOf(count), count > 0 ? style : Theme.muted())); + spans.add(Span.raw(" " + words + " ")); + } + + // ---- drawing ---- + + /** + * Columns of values stacked, the first series at the bottom, scaled to the highest column, with eighths of a cell + * at the top of each column and the scale at the left. + */ + static List<Line> stacked(List<long[]> series, List<Style> styles, int rows, int columns, int label) { + long[] totals = new long[columns]; + long max = 0; + for (long[] s : series) { + for (int c = 0; c < columns && c < s.length; c++) { + totals[c] += Math.max(0, s[c]); + } + } + for (long t : totals) { + max = Math.max(max, t); + } + List<Line> lines = new ArrayList<>(); + int cells = rows * 8; + for (int row = rows - 1; row >= 0; row--) { + List<Span> spans = new ArrayList<>(); + String scale = row == rows - 1 ? String.valueOf(max) : row == 0 ? "0" : ""; + spans.add(Span.styled(padLeft(scale, label - 2) + " β€", Theme.muted())); + for (int c = 0; c < columns; c++) { + long t = totals[c]; + int height = max == 0 || t == 0 ? 0 : (int) Math.max(1, Math.round((double) t * cells / max)); + int filled = Math.max(0, Math.min(8, height - row * 8)); + if (filled == 0) { + spans.add(Span.raw(" ")); + continue; + } + // the series at the top of what is filled in this cell + int at = row * 8 + filled - 1; + long cum = 0; + Style style = styles.get(styles.size() - 1); + for (int k = 0; k < series.size(); k++) { + long v = c < series.get(k).length ? Math.max(0, series.get(k)[c]) : 0; + cum += v; + if (v > 0 && Math.round((double) cum * height / t) > at) { + style = styles.get(k); + break; + } + } + spans.add(Span.styled(String.valueOf(BLOCKS.charAt(filled - 1)), style)); + } + lines.add(Line.from(spans)); + } + return lines; + } + + /** A sparkline of values (-1 is unknown, blank), relative to the highest. */ + static String spark(long[] values, long max) { + StringBuilder sb = new StringBuilder(); + for (long v : values) { + if (v < 0) { + sb.append(' '); + } else if (max <= 0 || v == 0) { + sb.append('β'); + } else { + sb.append(BLOCKS.charAt((int) Math.max(0, Math.min(7, Math.ceil(v * 8.0 / max) - 1)))); + } + } + return sb.toString(); + } + + /** + * The seconds as columns of several seconds each: the sum of the counts, or the highest value of a level (a mean, + * the in flight count). A column with no known second is -1. + */ + static long[] columnsOf(long[] seconds, int per, boolean highest) { + int columns = (seconds.length + per - 1) / per; + long[] answer = new long[columns]; + Arrays.fill(answer, -1); + for (int i = 0; i < seconds.length; i++) { + long v = seconds[i]; + if (v < 0) { + continue; + } + int c = i / per; + answer[c] = answer[c] < 0 ? v : highest ? Math.max(answer[c], v) : answer[c] + v; + } + return answer; + } + + private static long total(long... values) { + long sum = 0; + for (long v : values) { + sum += Math.max(0, v); + } + return sum; + } + + private static long sum(long[] values) { + return total(values); + } + + private static Style handledStyle() { + return Style.EMPTY.fg(Theme.ACCENT); + } + + private static String pad(String s, int width) { + return s.length() >= width ? s : s + " ".repeat(width - s.length()); + } + + private static String padLeft(String s, int width) { + return s.length() >= width ? s : " ".repeat(width - s.length()) + s; + } + + private static Block block(String title) { + return Block.builder().borderType(BorderType.ROUNDED).borders(Borders.ALL).title(title).build(); + } + + @Override + public void renderFooter(List<Span> spans) { + hint(spans, "ββ", "route"); + hint(spans, "w", "window " + (seconds() >= 120 ? seconds() / 60 + "m" : seconds() + "s")); + hint(spans, "Space", pausedAt >= 0 ? "follow" : "pause"); + if (pausedAt >= 0) { + hint(spans, "ββ", "back and forth"); + } + hint(spans, "Esc", pausedAt >= 0 ? "follow" : "back"); + } + + @Override + public SelectionContext getSelectionContext() { + List<Timeline.RouteSeries> routes = routes(ctx.findSelectedIntegration()); + if (routes.isEmpty()) { + return null; + } + List<String> items = routes.stream().map(Timeline.RouteSeries::route).toList(); + Integer sel = tableState.selected(); + return new SelectionContext("table", items, sel != null ? sel : -1, items.size(), "Routes"); + } + + @Override + public String description() { + return "A timeline of the integration: errors and route outcomes per second, with chaos rules as a layer"; + } + + @Override + public String getHelpText() { + return DocHelper.loadHelpText("timeline"); + } + + @Override + public JsonObject getTableDataAsJson() { + IntegrationInfo info = ctx.findSelectedIntegration(); + if (info == null) { + return null; + } + JsonObject result = new JsonObject(); + result.put("tab", "Timeline"); + result.put("seconds", seconds()); + result.put("endSecond", endSecond()); + result.put("paused", pausedAt >= 0); + result.put("order", "oldest first, one value per second; -1 is a second before the first sample"); + JsonArray causes = new JsonArray(); + for (Timeline.CauseSeries c : ctx.timeline.causes(info.pid, endSecond(), seconds())) { + JsonObject jo = new JsonObject(); + jo.put("label", c.label()); + if (c.rule() != null) { + jo.put("chaosRule", c.rule()); + } + jo.put("total", sum(c.values())); + jo.put("perSecond", array(c.values())); + causes.add(jo); + } + result.put("causes", causes); + JsonArray rows = new JsonArray(); + for (Timeline.RouteSeries r : routes(info)) { + JsonObject row = new JsonObject(); + row.put("route", r.route()); + row.put("completed", r.total(r.completed())); + row.put("recovered", r.total(r.recovered())); + row.put("handled", r.total(r.handled())); + row.put("failed", r.total(r.failed())); + row.put("downSeconds", r.total(r.down())); + JsonObject perSecond = new JsonObject(); + perSecond.put("completed", array(r.completed())); + perSecond.put("recovered", array(r.recovered())); + perSecond.put("handled", array(r.handled())); + perSecond.put("failed", array(r.failed())); + perSecond.put("inflight", array(r.inflight())); + perSecond.put("meanMillis", array(r.mean())); + row.put("perSecond", perSecond); + rows.add(row); + } + result.put("rows", rows); + result.put("totalRows", rows.size()); + Integer sel = tableState.selected(); + result.put("selectedIndex", sel != null ? sel : -1); + return result; + } + + private static JsonArray array(long[] values) { + JsonArray a = new JsonArray(); + for (long v : values) { + a.add(v); + } + return a; + } +} diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TuiIcons.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TuiIcons.java index 29821e0c3dba..99eed1942d26 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TuiIcons.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TuiIcons.java @@ -168,6 +168,7 @@ final class TuiIcons { static final String TAB_PROCESS = CLIPBOARD; static final String TAB_STARTUP = QUARKUS; static final String TAB_THREADS = "π§΅"; + static final String TAB_TIMELINE = "π"; static final String TAB_PRODUCERS = "π€"; static final String TAB_EVENTS = "π£"; static final String TAB_RECOVERY_TASKS = "π"; diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/timeline.md b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/timeline.md new file mode 100644 index 000000000000..d746c35f7de2 --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/timeline.md @@ -0,0 +1,65 @@ +# Timeline + +The last minute of the selected integration on one time axis, one column +per second: what went wrong (the cause) on top, and what it did to the +routes (the effect) under it. Read it left to right: "the stock service +answered 500 for ten seconds, redelivery saved the first orders, the rest +went to the dead letter channel, and it recovered by itself". + +The timeline is recorded from the moment the TUI watches the integration, +so it opens with the last minutes, not an empty chart. + +## Errors per second + +The errors of the integration (as the Errors tab lists them), stacked by +the kind of exception, with the HTTP status when there is one, such as +`HttpOperationFailedException 503`. An error counts once per exchange +that failed or that the error handler handled. A failure that a +redelivery recovered is not an error of the exchange: look for it in the +yellow of the routes. + +The errors are read while this screen or the Errors tab shows. The first +time, the errors the integration still keeps are put at their own second. + +With camel-chaos on, the faults of each chaos rule are a layer of their +own, marked π² with the id of the rule (`π² r1 error 503 (stock)`). An +error that a rule made is not counted again as an error, so an injected +fault and a real one stay apart. + +When nothing went wrong, the band is one line: `Errors/s none`. + +## Routes + +A line per route, one character per second. The height is the number of +exchanges done that second, relative to the busiest second of the route. +The colour is the worst that happened to them that second: + +- green: completed +- yellow: recovered by redelivery (completed after the error handler + redelivered it) +- orange: handled by the error handler, such as moved to the dead letter + channel +- red: failed + +A `Β·` is a second with no exchange, and a red `β` is a second a chaos +rule had the route down (`stall` or a lifecycle fault). At the right: the +exchanges per second over the window, and the worst count. + +## The selected route + +Its outcomes stacked per second (green at the bottom, then yellow, orange +and red), the mean processing time of the exchanges done each second, and +the exchanges in flight. The last line has the totals of the window. + +## Keys + +- **Up/Down** β Select a route for the detail +- **w** β The window: 1 minute, 2 minutes or 5 minutes (several seconds + a column when the screen is too narrow) +- **Space** β Pause, to read a moment while it scrolls away; Space or Esc + follows now again +- **Left/Right** β While paused, go back and forth in the 5 minutes kept + +The numbers come from the route statistics in the status file, as the +difference from one second to the next. An AI can read the same per +second series with `tui_get_table` on the Timeline tab. diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTabRenderTest.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTabRenderTest.java new file mode 100644 index 000000000000..9b09190c0e72 --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTabRenderTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands.tui; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import dev.tamboui.text.Span; +import dev.tamboui.tui.event.KeyEvent; +import org.apache.camel.util.json.JsonArray; +import org.apache.camel.util.json.JsonObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The Timeline screen ({@link TimelineTab}) of an integration in an outage of the stock service: for 15 seconds the + * orders go to the dead letter channel with a 503, and shipping has nothing to ship. + */ +class TimelineTabRenderTest { + + private static final long T0 = 1_800_000_000_000L; + + private final AtomicLong now = new AtomicLong(T0 + 59_000); + private MonitorContext ctx; + private IntegrationInfo info; + + @BeforeEach + void setUp() { + Theme.resetForTesting(); + info = new IntegrationInfo(); + info.pid = "1234"; + info.name = "shop"; + ctx = new MonitorContext(new AtomicReference<>(List.of(info)), new AtomicReference<>(List.of())); + ctx.selectedPid = "1234"; + long orders = 0; + long handled = 0; + long shipping = 0; + List<ErrorInfo> errors = new ArrayList<>(); + for (int s = 0; s < 60; s++) { + boolean outage = s >= 25 && s < 40; + orders += 2; + if (outage) { + for (int k = 0; k < 2; k++) { + handled++; + ErrorInfo e = new ErrorInfo(); + e.routeId = "orders"; + e.nodeId = "callStock"; + e.exceptionType = "org.apache.camel.http.base.HttpOperationFailedException"; + e.exceptionMessage = "HTTP operation failed invoking http://localhost:8765/stock with statusCode: 503"; + e.handled = true; + e.repeatCount = handled; + e.repeatFirstTimestamp = T0 + 25_000; + e.timestamp = T0 + s * 1000L + k * 400; + errors.add(0, e); + } + } else { + shipping += 2; + } + info.routes.clear(); + info.routes.add(route("orders", orders, handled)); + info.routes.add(route("shipping", shipping, 0)); + info.routes.add(route("dlq", handled, 0)); + ctx.timeline.record(info, T0 + s * 1000L); + } + // the error registry keeps the newest 3 of a kind, with the count of the kind + ctx.timeline.recordErrors("1234", errors.subList(0, 3), T0 + 59_000); + } + + private static RouteInfo route(String id, long total, long handled) { + RouteInfo r = new RouteInfo(); + r.routeId = id; + r.total = total; + r.failuresHandled = handled; + r.meanTime = 4; + return r; + } + + @Test + void theOutageReadsFromLeftToRight() { + TimelineTab tab = new TimelineTab(ctx, now::get); + String screen = TuiTestHelper.renderToString(tab, 160, 40); + System.out.println(screen); + + assertThat(screen).contains("Timeline β shop β last 60s") + .contains("Errors/s").contains("HttpOperationFailedException 503") + .contains(">> orders").contains("shipping").contains("dlq") + .contains("-60s").contains("-30s").contains("now") + .contains("orders β per second").contains("mean time").contains("in flight") + .contains("30 handled by the error handler"); + // shipping had nothing to ship in the outage: 15 quiet seconds + String shipping = screen.lines().filter(l -> l.contains(" shipping")).findFirst().orElseThrow(); + assertThat(shipping).contains("Β·Β·Β·Β·Β·Β·Β·Β·Β·Β·Β·Β·Β·Β·Β·"); + } + + @Test + void theTableForAnAiHasTheSeriesPerSecond() { + TimelineTab tab = new TimelineTab(ctx, now::get); + TuiTestHelper.renderToString(tab, 160, 40); + JsonObject table = tab.getTableDataAsJson(); + + assertThat(table.getString("tab")).isEqualTo("Timeline"); + JsonArray causes = (JsonArray) table.get("causes"); + assertThat(causes).hasSize(1); + // 3 entries kept, and the 27 before them spread from the first failure of the kind + assertThat(((JsonObject) causes.get(0)).getLong("total")).isEqualTo(30); + JsonArray rows = (JsonArray) table.get("rows"); + JsonObject orders = (JsonObject) rows.get(0); + assertThat(orders.getString("route")).isEqualTo("orders"); + assertThat(orders.getLong("handled")).isEqualTo(30); + // the first second has no rise yet + assertThat(orders.getLong("completed")).isEqualTo(118 - 30); + } + + @Test + void pauseKeepsTheMomentAndGoesBack() { + TimelineTab tab = new TimelineTab(ctx, now::get); + tab.handleKeyEvent(KeyEvent.ofChar(' ')); + assertThat(tab.isPaused()).isTrue(); + now.addAndGet(10_000); + String screen = TuiTestHelper.renderToString(tab, 160, 40); + assertThat(screen).contains("paused, 10s ago"); + List<Span> hints = new ArrayList<>(); + tab.renderFooter(hints); + assertThat(text(hints)).contains("follow").contains("back and forth"); + + assertThat(tab.handleEscape()).isTrue(); + assertThat(tab.isPaused()).isFalse(); + } + + @Test + void theWindowGrowsToFiveMinutes() { + TimelineTab tab = new TimelineTab(ctx, now::get); + tab.handleKeyEvent(KeyEvent.ofChar('w')); + tab.handleKeyEvent(KeyEvent.ofChar('w')); + assertThat(tab.seconds()).isEqualTo(300); + String screen = TuiTestHelper.renderToString(tab, 160, 40); + assertThat(screen).contains("last 5m").contains("-5m").contains("s a column"); + } + + @Test + void aQuietIntegrationIsACalmScreen() { + IntegrationInfo quiet = new IntegrationInfo(); + quiet.pid = "77"; + quiet.name = "quiet"; + RouteInfo r = route("timer", 0, 0); + quiet.routes.add(r); + MonitorContext qctx = new MonitorContext(new AtomicReference<>(List.of(quiet)), new AtomicReference<>(List.of())); + qctx.selectedPid = "77"; + for (int s = 0; s < 60; s++) { + qctx.timeline.record(quiet, T0 + s * 1000L); + } + qctx.timeline.recordErrors("77", List.of(), T0 + 59_000); + String screen = TuiTestHelper.renderToString(new TimelineTab(qctx, now::get), 160, 40); + + assertThat(screen).contains("Errors/s none").contains("idle"); + } + + private static String text(List<Span> spans) { + StringBuilder sb = new StringBuilder(); + spans.forEach(s -> sb.append(s.content())); + return sb.toString(); + } +} diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTest.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTest.java new file mode 100644 index 000000000000..bbaf0e7cc8c4 --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/TimelineTest.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands.tui; + +import java.util.List; +import java.util.Set; + +import org.apache.camel.util.json.JsonObject; +import org.apache.camel.util.json.Jsoner; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The timeline (CAMEL-26691) worked out from the samples of the route counters, the errors and the chaos rules. + */ +class TimelineTest { + + private static final long T0 = 1_800_000_000_000L; + private static final long S0 = T0 / 1000; + + private final Timeline timeline = new Timeline(); + + private static IntegrationInfo info(long total, long failed, long handled, long redeliveries, long mean) { + IntegrationInfo info = new IntegrationInfo(); + info.pid = "1234"; + info.name = "shop"; + RouteInfo r = new RouteInfo(); + r.routeId = "orders"; + r.total = total; + r.failed = failed; + r.failuresHandled = handled; + r.redeliveries = redeliveries; + r.meanTime = mean; + r.inflight = 2; + info.routes.add(r); + return info; + } + + private Timeline.RouteSeries orders(long endSecond, int seconds) { + List<Timeline.RouteSeries> routes = timeline.routes("1234", List.of("orders"), endSecond, seconds); + assertThat(routes).hasSize(1); + return routes.get(0); + } + + @Test + void theOutcomesOfASecondAreTheRiseOfTheCounters() { + timeline.record(info(10, 0, 0, 0, 4), T0); + // 3 more exchanges: one handled by the error handler, one recovered by redelivery, one completed + timeline.record(info(13, 0, 1, 1, 6), T0 + 1000); + + Timeline.RouteSeries r = orders(S0 + 1, 3); + // the second before the first sample, the first sample (no rise yet), then the rise + assertThat(r.completed()).containsExactly(-1, -1, 1); + assertThat(r.recovered()).containsExactly(-1, -1, 1); + assertThat(r.handled()).containsExactly(-1, -1, 1); + assertThat(r.failed()).containsExactly(-1, -1, 0); + assertThat(r.inflight()).containsExactly(-1, 2, 2); + // the mean of the 3 exchanges of that second: (13 * 6 - 10 * 4) / 3 + assertThat(r.mean()[2]).isEqualTo(12); + assertThat(r.quiet()).isFalse(); + } + + @Test + void aSlowRefreshIsSpreadOverItsSeconds() { + timeline.record(info(0, 0, 0, 0, 0), T0); + timeline.record(info(7, 1, 0, 0, 0), T0 + 3000); + + Timeline.RouteSeries r = orders(S0 + 3, 3); + // 6 completed and 1 failed over 3 seconds: the later seconds take the remainder + assertThat(r.completed()).containsExactly(2, 2, 2); + assertThat(r.failed()).containsExactly(0, 0, 1); + } + + @Test + void aResetOfTheStatisticsIsNoNegativeRate() { + timeline.record(info(50, 3, 0, 0, 0), T0); + timeline.record(info(2, 0, 0, 0, 0), T0 + 1000); + + Timeline.RouteSeries r = orders(S0 + 1, 1); + assertThat(r.completed()).containsExactly(0); + assertThat(r.failed()).containsExactly(0); + } + + @Test + void theErrorsArePerSecondByKindWithoutTheOnesChaosMade() { + ErrorInfo first = error("orders", "callStock", "HTTP operation failed with statusCode: 503", 1, T0 - 2000); + ErrorInfo chaos = error("orders", "callStock", "HTTP operation failed with statusCode: 500", 1, T0 - 1000); + chaos.properties.put(ChaosMarks.PROPERTY, "r1"); + timeline.recordErrors("1234", List.of(first, chaos), T0); + // the same kind failed 3 more times; the registry keeps the newest entries of a kind + ErrorInfo second = error("orders", "callStock", "HTTP operation failed with statusCode: 503", 3, T0 + 1500); + ErrorInfo third = error("orders", "callStock", "HTTP operation failed with statusCode: 503", 4, T0 + 2500); + timeline.recordErrors("1234", List.of(third, second, first, chaos), T0 + 3000); + + List<Timeline.CauseSeries> causes = timeline.causes("1234", S0 + 2, 5); + assertThat(causes).hasSize(1); + assertThat(causes.get(0).label()).isEqualTo("HttpOperationFailedException 503"); + assertThat(causes.get(0).rule()).isNull(); + // the first at its own second, then 3 new ones: the entry at +1.5s, and the rest at the newest (+2.5s) + assertThat(causes.get(0).values()).containsExactly(1, 0, 0, 1, 2); + assertThat(timeline.errorsSeen("1234")).isTrue(); + } + + @Test + void theFaultsOfAChaosRuleAndTheSecondsItHadARouteDown() throws Exception { + IntegrationInfo info = info(0, 0, 0, 0, 0); + info.chaos = ChaosInfo.parse((JsonObject) Jsoner.deserialize(""" + {"enabled": true, + "rules": [{"id": "r1", "version": 1, "state": "active", + "rule": "endpoint=http://localhost:8765/stock* error status=503 rate=0.2", + "matches": 10, "fired": {"send": 3}, "faultsPerSecond": [0, 2, 1]}, + {"id": "r3", "version": 1, "state": "active", + "rule": "route=orders stall duration=20s", + "matches": 1, "fired": {"lifecycle": 1}, + "lifecycle": {"action": "suspend", "route": "orders", "state": "down"}}], + "report": {"verdict": "PASS", "routes": [], "recentFaults": []}} + """)); + timeline.record(info, T0); + + List<Timeline.CauseSeries> causes = timeline.causes("1234", S0, 3); + assertThat(causes).hasSize(1); + assertThat(causes.get(0).rule()).isEqualTo("r1"); + assertThat(causes.get(0).label()).isEqualTo("r1 error status 503 (stock)"); + assertThat(causes.get(0).values()).containsExactly(0, 2, 1); + assertThat(orders(S0, 3).down()).containsExactly(0, 0, 1); + } + + @Test + void aStoppedIntegrationIsForgotten() { + timeline.record(info(1, 0, 0, 0, 0), T0); + assertThat(timeline.firstSecond("1234")).isEqualTo(S0); + timeline.retain(Set.of("999")); + assertThat(timeline.firstSecond("1234")).isEqualTo(-1); + assertThat(timeline.routes("1234", List.of("orders"), S0, 60)).isEmpty(); + } + + private static ErrorInfo error(String route, String node, String message, long repeat, long timestamp) { + ErrorInfo e = new ErrorInfo(); + e.routeId = route; + e.nodeId = node; + e.exceptionType = "org.apache.camel.http.base.HttpOperationFailedException"; + e.exceptionMessage = message; + e.repeatCount = repeat; + e.timestamp = timestamp; + return e; + } +}
