This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch fix/CAMEL-24805-check-simple-in-answers in repository https://gitbox.apache.org/repos/asf/camel.git
commit af14f580da6e3c4a3a04537fad44843a3f5192b5 Author: Claus Ibsen <[email protected]> AuthorDate: Thu Sep 17 21:43:00 2026 +0200 CAMEL-24805: camel-jbang - the TUI AI panel checks the simple expressions of an answer before showing it, and gives the model one turn to fix them What the model writes goes through the validator before camel_write_file writes it; what it says in the answer did not, and the answer is what the user copies. A small local model puts the operator inside the placeholder (${header.user ?: 'Guest'}) even after seeing the correct example. AnswerChecks in camel-jbang-core extracts the simple expressions of an answer (the ${...} placeholders of the text and of the non-YAML code blocks, only those starting with a simple function or value so a Maven property is left alone; the simple: values and log messages of YAML blocks through SimpleChecks) and validates them with the catalog. The panel sends an invalid answer back once with what is wrong and the rule, within the tool-call budget, shows the corrected answer with '1 correction' in the byline, and when the model does not fix it shows the answer with a Simple check line under it. The exchange is in the AI log. Co-Authored-By: Claude Fable 5.1 <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../modules/ROOT/pages/camel-jbang-tui.adoc | 11 + .../dsl/jbang/core/commands/ai/AnswerChecks.java | 221 +++++++++++++++++++++ .../jbang/core/commands/ai/AnswerChecksTest.java | 123 ++++++++++++ .../camel/dsl/jbang/core/commands/tui/AiPanel.java | 79 +++++++- .../dsl/jbang/core/commands/tui/AiPanelTest.java | 101 ++++++++++ 5 files changed, 526 insertions(+), 9 deletions(-) diff --git a/docs/user-manual/modules/ROOT/pages/camel-jbang-tui.adoc b/docs/user-manual/modules/ROOT/pages/camel-jbang-tui.adoc index 3cc8ee23226b..5ae3ee5b0a8e 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-jbang-tui.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-jbang-tui.adoc @@ -1133,6 +1133,17 @@ yellow from 10 requests, and the total time yellow from 30 seconds and orange fr that runs into the limit is usually a tool that makes the model guess, not a weak model: the AI log shows which one. +What the model says is checked like what it writes. A file goes through the validator before +`camel_write_file` writes it; the answer shown in the chat is what you copy, so its simple expressions +are checked against the catalog too: the `${...}` placeholders in the text and in Java or XML code +blocks, and the `simple:` values and log messages of YAML blocks. A small model tends to put the +operator inside the placeholder (`${header.user ?: 'Guest'}` instead of `${header.user} ?: 'Guest'`). +When the check finds such an expression the panel sends the answer back to the model once, with what is +wrong and the rule, and shows the corrected answer with `1 correction` in the byline. When the model +does not fix it the answer is shown as it is with a *Simple check* line under it naming the expression +and the error, and the AI log has the exchange. Placeholders that are not simple (a Maven +`${camel-version}`, a shell variable) are left alone. + Press *Ctrl+U* while the AI panel is open to toggle the AI Usage view. It shows token consumption from the embedded AI prompt (*TUI ask*) and from LLM calls made by the monitored integration (*integration*), the latter taken from OpenTelemetry GenAI spans when observability is enabled on the integration. The diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/AnswerChecks.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/AnswerChecks.java new file mode 100644 index 000000000000..e67122542d51 --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/AnswerChecks.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands.ai; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import org.apache.camel.catalog.CamelCatalog; +import org.apache.camel.catalog.DefaultCamelCatalog; +import org.apache.camel.catalog.LanguageValidationResult; +import org.apache.camel.tooling.model.LanguageModel; + +/** + * The checks of what an AI model says, as opposed to what it writes: a file goes through the validator before it is + * written, an answer shown in a chat does not, and it is what the user copies. The simple expressions of an answer are + * checked against the catalog, the way {@code camel_validate_source} checks a route: the {@code ${...}} placeholders of + * the text and of any code block that is not YAML, and the {@code simple:} values and log messages of the YAML blocks. + * <p> + * A placeholder is only checked when it starts with a simple function or value (body, header, exchangeProperty, random, + * date, ...), so a Maven {@code ${camel-version}} or a shell variable in the same answer is left alone. What the + * catalog cannot judge is skipped as in {@link SimpleChecks}: a function of a language that is not on the classpath, a + * property placeholder used as a logical operand. + */ +public final class AnswerChecks { + + /** A simple expression of an answer that the catalog rejects: what was written, and why. */ + public record Problem(String expression, String error) { + + /** {@code ${header.user ?: 'Guest'}: Unexpected token ?: at location 13}. */ + public String message() { + return expression + ": " + error; + } + } + + private static volatile CamelCatalog defaultCatalog; + private static volatile Set<String> roots; + + private AnswerChecks() { + } + + /** The problems of the simple expressions of an answer, checked against the default catalog. */ + public static List<Problem> checkSimple(String markdown) { + return checkSimple(markdown, defaultCatalog()); + } + + /** The problems of the simple expressions of an answer, checked against the given catalog. */ + public static List<Problem> checkSimple(String markdown, CamelCatalog catalog) { + if (markdown == null || markdown.isBlank() || catalog == null) { + return List.of(); + } + Map<String, Problem> problems = new LinkedHashMap<>(); + StringBuilder text = new StringBuilder(); + StringBuilder block = null; + boolean yaml = false; + for (String line : markdown.split("\n", -1)) { + String trimmed = line.trim(); + if (trimmed.startsWith("```")) { + if (block == null) { + String language = trimmed.substring(3).trim().toLowerCase(Locale.ROOT); + yaml = language.equals("yaml") || language.equals("yml"); + block = new StringBuilder(); + } else { + if (yaml) { + for (String error : SimpleChecks.validateYamlSimple(block.toString(), catalog)) { + problems.putIfAbsent(error, new Problem("the YAML block", error)); + } + } else { + text.append(block).append('\n'); + } + block = null; + } + continue; + } + if (block != null) { + block.append(line).append('\n'); + } else { + text.append(line).append('\n'); + } + } + if (block != null) { + // an unterminated fence: the model ran out of tokens, judge what is there as text + text.append(block); + } + for (String placeholder : placeholders(text.toString())) { + if (problems.containsKey(placeholder) || !isSimple(placeholder, catalog) + || SimpleChecks.hasPlaceholderAsLogicalOperand(placeholder)) { + continue; + } + try { + LanguageValidationResult result = catalog.validateLanguageExpression(null, "simple", placeholder); + if (!result.isSuccess()) { + String error = result.getShortError() != null ? result.getShortError() : result.getError(); + if (error != null && !SimpleChecks.isMissingDependency(error)) { + problems.put(placeholder, new Problem(placeholder, error)); + } + } + } catch (Exception e) { + // best effort: what the catalog cannot judge is not reported + } + } + return new ArrayList<>(problems.values()); + } + + /** + * The {@code ${...}} placeholders of a text, braces balanced so {@code ${header.${header.key}}} is one placeholder, + * in the order they appear. + */ + static List<String> placeholders(String text) { + List<String> answer = new ArrayList<>(); + int i = 0; + while (i < text.length() - 1) { + if (text.charAt(i) == '$' && text.charAt(i + 1) == '{') { + int depth = 0; + int end = -1; + for (int j = i + 1; j < text.length(); j++) { + char ch = text.charAt(j); + if (ch == '{') { + depth++; + } else if (ch == '}') { + depth--; + if (depth == 0) { + end = j; + break; + } + } else if (ch == '\n' && depth == 1) { + // a placeholder does not span lines; an unclosed one is prose + break; + } + } + if (end < 0) { + i += 2; + continue; + } + answer.add(text.substring(i, end + 1)); + i = end + 1; + } else { + i++; + } + } + return answer; + } + + /** Whether the placeholder starts with a simple function or value, so the catalog is the judge of it. */ + static boolean isSimple(String placeholder, CamelCatalog catalog) { + String content = placeholder.substring(2, placeholder.length() - 1).strip(); + int end = 0; + while (end < content.length()) { + char ch = content.charAt(end); + if (!Character.isLetterOrDigit(ch) && ch != '-' && ch != '_') { + break; + } + end++; + } + if (end == 0) { + return false; + } + return roots(catalog).contains(content.substring(0, end)); + } + + /** The first word of every simple function name: header for header.name, date for date:command:pattern. */ + private static Set<String> roots(CamelCatalog catalog) { + Set<String> answer = roots; + if (answer == null) { + answer = new HashSet<>(); + LanguageModel simple = catalog.languageModel("simple"); + if (simple != null && simple.getFunctions() != null) { + for (LanguageModel.LanguageFunctionModel fn : simple.getFunctions()) { + String name = fn.getName(); + if (name != null) { + int end = 0; + while (end < name.length()) { + char ch = name.charAt(end); + if (!Character.isLetterOrDigit(ch) && ch != '-' && ch != '_') { + break; + } + end++; + } + if (end > 0) { + answer.add(name.substring(0, end)); + } + } + } + } + roots = answer; + } + return answer; + } + + private static CamelCatalog defaultCatalog() { + CamelCatalog answer = defaultCatalog; + if (answer == null) { + synchronized (AnswerChecks.class) { + answer = defaultCatalog; + if (answer == null) { + answer = new DefaultCamelCatalog(); + defaultCatalog = answer; + } + } + } + return answer; + } +} diff --git a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/AnswerChecksTest.java b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/AnswerChecksTest.java new file mode 100644 index 000000000000..3a525018116b --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/AnswerChecksTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands.ai; + +import java.util.List; + +import org.apache.camel.catalog.CamelCatalog; +import org.apache.camel.catalog.DefaultCamelCatalog; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * CAMEL-24805: the simple expressions an AI model writes in an answer are checked the way a file is, since the answer + * is what the user copies and where a small model puts the operator inside the placeholder. + */ +class AnswerChecksTest { + + private static final CamelCatalog CATALOG = new DefaultCamelCatalog(); + + @Test + void theOperatorInsideThePlaceholderIsFoundInProseAndInlineCode() { + String answer = "Use `${header.user ?: 'Guest'}` to fall back to a guest name, and log it with " + + "`${header.user} ?: 'Guest'` between placeholders is what Camel expects."; + List<AnswerChecks.Problem> problems = AnswerChecks.checkSimple(answer, CATALOG); + assertThat(problems).hasSize(1); + assertThat(problems.get(0).expression()).isEqualTo("${header.user ?: 'Guest'}"); + assertThat(problems.get(0).error()).isNotBlank(); + assertThat(problems.get(0).message()).startsWith("${header.user ?: 'Guest'}: "); + // the same mistake twice is one problem + assertThat(AnswerChecks.checkSimple(answer + "\nAgain: ${header.user ?: 'Guest'}", CATALOG)).hasSize(1); + } + + @Test + void correctSimpleAndForeignPlaceholdersPass() { + String answer = """ + The body is ${body}, the header ${header.user}, a random number ${random(1,10)} and today + ${date:now:yyyy-MM-dd}; nested works too: ${header.${header.key}}. + The Maven property ${camel-version} and the shell variable ${HOME} are not simple. + A function of another language, ${jsonpath($.name)}, needs its dependency at runtime. + A property placeholder used as a predicate, {{enabled}} && ${body} != null, is judged at runtime. + """; + assertThat(AnswerChecks.checkSimple(answer, CATALOG)).isEmpty(); + assertThat(AnswerChecks.checkSimple(null, CATALOG)).isEmpty(); + assertThat(AnswerChecks.checkSimple("no expressions here", CATALOG)).isEmpty(); + } + + @Test + void yamlBlocksAreCheckedAsRoutesAndOtherBlocksAsText() { + String answer = """ + Here is the route: + + ```yaml + - route: + from: + uri: timer:tick + steps: + - setBody: + expression: + simple: + expression: "${header.user ?: 'Guest'}" + - log: + message: "Hello ${body}" + ``` + + And in Java: + + ```java + from("timer:tick").setBody(simple("${header.user ?: 'Guest'}")); + ``` + """; + List<AnswerChecks.Problem> problems = AnswerChecks.checkSimple(answer, CATALOG); + assertThat(problems).hasSize(2); + assertThat(problems.get(0).expression()).isEqualTo("the YAML block"); + assertThat(problems.get(0).error()).startsWith("Line 8: Simple syntax error: "); + assertThat(problems.get(1).expression()).isEqualTo("${header.user ?: 'Guest'}"); + + String valid = """ + ```yaml + - route: + from: + uri: timer:tick + steps: + - setBody: + expression: + simple: + expression: "${header.user} ?: 'Guest'" + ``` + """; + assertThat(AnswerChecks.checkSimple(valid, CATALOG)).isEmpty(); + } + + @Test + void placeholdersAreBalancedAndStayOnOneLine() { + assertThat(AnswerChecks.placeholders("a ${body} b ${header.${header.key}} c ${x")) + .containsExactly("${body}", "${header.${header.key}}"); + assertThat(AnswerChecks.placeholders("${header.a\n}")).isEmpty(); + assertThat(AnswerChecks.isSimple("${header.user}", CATALOG)).isTrue(); + assertThat(AnswerChecks.isSimple("${date-with-timezone:now:UTC:yyyy}", CATALOG)).isTrue(); + assertThat(AnswerChecks.isSimple("${camel-version}", CATALOG)).isFalse(); + assertThat(AnswerChecks.isSimple("${}", CATALOG)).isFalse(); + } + + @Test + void theDefaultCatalogIsUsedWhenNoneIsGiven() { + assertThat(AnswerChecks.checkSimple("${header.user ?: 'Guest'}")).hasSize(1); + assertThat(AnswerChecks.checkSimple("${header.user} ?: 'Guest'")).isEmpty(); + } +} diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/AiPanel.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/AiPanel.java index 8a07a06a90c6..e105802b3fe9 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/AiPanel.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/AiPanel.java @@ -74,6 +74,7 @@ import dev.tamboui.widgets.table.Row; import dev.tamboui.widgets.table.Table; import dev.tamboui.widgets.table.TableState; import org.apache.camel.dsl.jbang.core.commands.LlmClient; +import org.apache.camel.dsl.jbang.core.commands.ai.AnswerChecks; import org.apache.camel.dsl.jbang.core.common.ExampleHelper; import org.apache.camel.dsl.jbang.core.common.Printer; import org.apache.camel.util.json.JsonObject; @@ -334,14 +335,20 @@ class AiPanel { * {@code toolCalls} TUI tool calls the model made. */ record ConversationEntry(AiRole role, String text, long elapsedMs, long aiMs, long toolMs, int toolCalls, - int totalTokens, int requests, boolean limitReached, int contextPercent) { + int totalTokens, int requests, boolean limitReached, int contextPercent, int corrections, String note) { ConversationEntry(AiRole role, String text) { - this(role, text, -1, 0, 0, 0, 0, 0, false, 0); + this(role, text, -1, 0, 0, 0, 0, 0, false, 0, 0, null); } ConversationEntry(AiRole role, String text, long elapsedMs, long aiMs, long toolMs, int toolCalls, int totalTokens) { - this(role, text, elapsedMs, aiMs, toolMs, toolCalls, totalTokens, 0, false, 0); + this(role, text, elapsedMs, aiMs, toolMs, toolCalls, totalTokens, 0, false, 0, 0, null); + } + + ConversationEntry(AiRole role, String text, long elapsedMs, long aiMs, long toolMs, int toolCalls, + int totalTokens, int requests, boolean limitReached, int contextPercent) { + this(role, text, elapsedMs, aiMs, toolMs, toolCalls, totalTokens, requests, limitReached, contextPercent, 0, + null); } /** "5.2s" or, when tools were called, "5.2s, ai 4.1s, tools 1.1s/3". */ @@ -374,6 +381,9 @@ class AiPanel { if (requests > 1) { sb.append(" · ").append(requests).append(" requests"); } + if (corrections > 0) { + sb.append(" · ").append(corrections).append(corrections == 1 ? " correction" : " corrections"); + } if (totalTokens > 0) { sb.append(" · ").append(LlmClient.formatTokens(totalTokens)).append(" tokens"); } @@ -1495,6 +1505,7 @@ class AiPanel { long turnToolMs = 0; int turnToolCalls = 0; int turnRequests = 0; + int turnCorrections = 0; for (int i = 0; i < MAX_ITERATIONS; i++) { if (Thread.interrupted()) { throw new InterruptedException(); @@ -1565,8 +1576,19 @@ class AiPanel { } messages.add(LlmClient.Message.toolResults(results)); } else { + // what the model says is checked like what it writes: an invalid simple expression in the answer + // gets one correction turn, since the answer is what the user copies (CAMEL-24805) + List<AnswerChecks.Problem> problems = AnswerChecks.checkSimple(response.text()); + if (!problems.isEmpty() && turnCorrections == 0 && i < MAX_ITERATIONS - 1) { + turnCorrections++; + String request = correctionRequest(problems); + messages.add(LlmClient.Message.assistantWithToolCalls(response.text(), List.of())); + messages.add(LlmClient.Message.user(request)); + log(LogLevel.RESULT, "Answer check: invalid simple expression, asking for a fix", request); + continue; + } completeTurn(response.text(), false, totalUsage, turnAiMs, turnToolMs, turnToolCalls, turnRequests, - messages); + turnCorrections, problems, messages); return; } } @@ -1603,8 +1625,9 @@ class AiPanel { } } if (wrapUp != null && wrapUp.text() != null && !wrapUp.text().isBlank()) { + // no round trip left for a correction: the answer is shown with what the check found completeTurn(wrapUp.text(), true, totalUsage, turnAiMs, turnToolMs, turnToolCalls, turnRequests, - messages); + turnCorrections, AnswerChecks.checkSimple(wrapUp.text()), messages); return; } @@ -1632,17 +1655,20 @@ class AiPanel { */ private void completeTurn( String text, boolean wrapUp, LlmClient.TokenUsage totalUsage, long turnAiMs, long turnToolMs, - int turnToolCalls, int turnRequests, List<LlmClient.Message> messages) { + int turnToolCalls, int turnRequests, int corrections, List<AnswerChecks.Problem> problems, + List<LlmClient.Message> messages) { sessionTotalTokens += totalUsage.totalTokens(); if (text != null && !text.isBlank()) { long elapsed = System.currentTimeMillis() - thinkingStartTime; + String note = problems.isEmpty() ? null : answerNote(problems); ConversationEntry entry = new ConversationEntry( AiRole.ASSISTANT, text, elapsed, turnAiMs, turnToolMs, turnToolCalls, - totalUsage.totalTokens(), turnRequests, wrapUp, contextFillPercent()); + totalUsage.totalTokens(), turnRequests, wrapUp, contextFillPercent(), corrections, note); conversation.add(entry); turnTimings.add(new long[] { turnAiMs, turnToolMs, turnToolCalls }); String label = wrapUp ? "Response after reaching the tool call limit (" : "Response ("; - log(LogLevel.RESPONSE, label + entry.byline() + describeCacheSignal(totalUsage) + ")", text); + log(LogLevel.RESPONSE, label + entry.byline() + describeCacheSignal(totalUsage) + ")", + note == null ? text : text + "\n\n" + note); } else { String err = "Empty response from LLM."; conversation.add(new ConversationEntry(AiRole.ERROR, err)); @@ -1805,6 +1831,36 @@ class AiPanel { return millis < 1000 ? millis + "ms" : formatSeconds(millis); } + /** + * The one correction turn a model gets when its answer has an invalid simple expression: what is wrong, the rule it + * broke, and to send the whole answer again. Worded so the fake clients of the tests and the log recognise it. + */ + static String correctionRequest(List<AnswerChecks.Problem> problems) { + StringBuilder sb = new StringBuilder("Your answer contains "); + sb.append(problems.size() == 1 ? "an invalid simple expression" : "invalid simple expressions").append(":\n"); + for (AnswerChecks.Problem p : problems) { + sb.append("- ").append(p.message()).append('\n'); + } + sb.append("Rule: functions and values go inside ${...}, operators go between placeholders with spaces:") + .append(" ${header.user} ?: 'Guest', ${header.a} == 'b', ${body} != null.") + .append(" Send the whole answer again with the expressions fixed; do not call a tool."); + return sb.toString(); + } + + /** The line under an answer whose simple expressions the model did not fix, so the user is not misled. */ + static String answerNote(List<AnswerChecks.Problem> problems) { + StringBuilder sb = new StringBuilder("**Simple check:** "); + if (problems.size() == 1) { + sb.append("`").append(problems.get(0).expression()).append("` is invalid: ").append(problems.get(0).error()); + } else { + sb.append(problems.size()).append(" expressions are invalid:"); + for (AnswerChecks.Problem p : problems) { + sb.append("\n- `").append(p.expression()).append("`: ").append(p.error()); + } + } + return sb.toString(); + } + private static String summarize(String text, int max) { if (text == null) { return ""; @@ -2391,7 +2447,12 @@ class AiPanel { for (ConversationEntry entry : conversation) { switch (entry.role()) { case USER -> md.append("> ").append(entry.text().replace("\n", "\n> ")).append("\n\n"); - case ASSISTANT -> md.append(toHardBreaks(entry.text())).append("\n\n"); + case ASSISTANT -> { + md.append(toHardBreaks(entry.text())).append("\n\n"); + if (entry.note() != null) { + md.append(entry.note()).append("\n\n"); + } + } case ERROR -> md.append("**Error:** ").append(entry.text()).append("\n\n"); case SYSTEM -> md.append(toHardBreaks(entry.text())).append("\n\n"); } diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/AiPanelTest.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/AiPanelTest.java index b5764201ec26..4074e7a8b27d 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/AiPanelTest.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/AiPanelTest.java @@ -1080,6 +1080,107 @@ class AiPanelTest { } } + // ---- the simple expressions of an answer are checked (CAMEL-24805) ---- + + @Test + void anInvalidSimpleExpressionInTheAnswerGetsOneCorrectionTurn() throws Exception { + AiPanel panel = new AiPanel(); + panel.setToolRegistryForTesting(new TuiToolRegistry(null)); + CorrectingLlmClient client = new CorrectingLlmClient(); + panel.setClientForTesting(client); + panel.open(); + type(panel, "how do I fall back to a guest name"); + panel.handleKeyEvent(KeyEvent.ofKey(KeyCode.ENTER, KeyModifiers.NONE)); + await().atMost(10, TimeUnit.SECONDS).until(() -> !panel.isAgentThreadRunningForTesting()); + + AiPanel.ConversationEntry last = panel.conversationForTesting().get(panel.conversationForTesting().size() - 1); + assertEquals(AiRole.ASSISTANT, last.role(), last.text()); + assertTrue(client.sawCorrectionRequest, "the model must be told what is invalid and asked to fix the answer"); + assertEquals(2, client.answers, "the first answer and the corrected one"); + assertTrue(last.text().contains("${header.user} ?: 'Guest'"), last.text()); + assertFalse(last.text().contains("${header.user ?: 'Guest'}"), last.text()); + assertNull(last.note(), "a corrected answer is shown as it is"); + assertEquals(1, last.corrections()); + assertEquals(2, last.requests()); + assertTrue(last.byline().contains("2 requests · 1 correction"), last.byline()); + assertTrue(panel.conversationForTesting().stream().noneMatch(e -> e.role() == AiRole.ERROR)); + } + + @Test + void anAnswerThatStaysWrongIsShownWithWhatTheCheckFound() throws Exception { + AiPanel panel = new AiPanel(); + panel.setToolRegistryForTesting(new TuiToolRegistry(null)); + CorrectingLlmClient client = new CorrectingLlmClient(); + client.fixes = false; + panel.setClientForTesting(client); + panel.open(); + type(panel, "how do I fall back to a guest name"); + panel.handleKeyEvent(KeyEvent.ofKey(KeyCode.ENTER, KeyModifiers.NONE)); + await().atMost(10, TimeUnit.SECONDS).until(() -> !panel.isAgentThreadRunningForTesting()); + + AiPanel.ConversationEntry last = panel.conversationForTesting().get(panel.conversationForTesting().size() - 1); + assertEquals(AiRole.ASSISTANT, last.role(), last.text()); + assertEquals(2, client.answers, "one correction turn, never a second"); + assertTrue(last.text().contains("${header.user ?: 'Guest'}"), last.text()); + assertNotNull(last.note(), "the user must see that the expression is invalid"); + assertTrue(last.note().startsWith("**Simple check:** `${header.user ?: 'Guest'}` is invalid: "), last.note()); + assertEquals(1, last.corrections()); + } + + @Test + void aCorrectAnswerIsNotSentBack() throws Exception { + AiPanel panel = new AiPanel(); + panel.setToolRegistryForTesting(new TuiToolRegistry(null)); + CorrectingLlmClient client = new CorrectingLlmClient(); + client.firstAnswerIsCorrect = true; + panel.setClientForTesting(client); + panel.open(); + type(panel, "how do I fall back to a guest name"); + panel.handleKeyEvent(KeyEvent.ofKey(KeyCode.ENTER, KeyModifiers.NONE)); + await().atMost(10, TimeUnit.SECONDS).until(() -> !panel.isAgentThreadRunningForTesting()); + + AiPanel.ConversationEntry last = panel.conversationForTesting().get(panel.conversationForTesting().size() - 1); + assertEquals(1, client.answers); + assertFalse(client.sawCorrectionRequest); + assertEquals(0, last.corrections()); + assertNull(last.note()); + assertFalse(last.byline().contains("correction"), last.byline()); + } + + /** Answers with the operator inside the placeholder, and fixes it when asked, like a small local model. */ + private static final class CorrectingLlmClient extends LlmClient { + + volatile boolean sawCorrectionRequest; + volatile int answers; + volatile boolean fixes = true; + volatile boolean firstAnswerIsCorrect; + + CorrectingLlmClient() { + withModel("test-model"); + withApiType(ApiType.openai); + } + + @Override + public boolean detectEndpoint() { + return true; + } + + @Override + public ChatResponse chatWithTools(String systemPrompt, List<Message> messages, List<ToolDef> tools) { + answers++; + Message last = messages.get(messages.size() - 1); + boolean correction = last.content() != null && last.content().contains("invalid simple expression"); + if (correction) { + sawCorrectionRequest = true; + } + boolean correct = firstAnswerIsCorrect || (correction && fixes); + String text = correct + ? "Use `${header.user} ?: 'Guest'` as the expression: the elvis operator goes between placeholders." + : "Use `${header.user ?: 'Guest'}` as the expression to fall back to Guest."; + return new ChatResponse(text, List.of(), "stop", false, TokenUsage.EMPTY); + } + } + /** Always asks for the same tool call, like a model stuck on a failing send. */ private static final class LoopingLlmClient extends LlmClient {
