This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 3646c6c7536e CAMEL-23688: camel-jbang - Tests for the questions of
camel ask (#27981)
3646c6c7536e is described below
commit 3646c6c7536e11c3c8595a7ab17d2ead586474bd
Author: Claus Ibsen <[email protected]>
AuthorDate: Sat Oct 17 00:34:38 2026 +0200
CAMEL-23688: camel-jbang - Tests for the questions of camel ask (#27981)
Tests for camel ask with a question (34% covered), against FakeLlmServer
playing Ollama: the answer, the tools the model calls and their results
in the next question, a failed call of the LLM, the maximum iterations
and the stats. FakeLlmServer can now reply in sequence, such as a tool
call and then the answer. They found these bugs, fixed here:
* camel ask exited with 0 without an answer when the call of the LLM
failed, such as with an error of the model or a refused connection:
the error answer of the client was taken as the final answer.
* The model in use, the integration, the note about a small model and
the output of --show-tools and --show-stats were printed on stdout
with the answer; they go to stderr, as with camel explain.
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
---
.../ROOT/pages/camel-4x-upgrade-guide-4_24.adoc | 7 +
.../modules/ROOT/pages/camel-jbang-ai.adoc | 4 +
.../apache/camel/dsl/jbang/core/commands/Ask.java | 34 +++--
.../camel/dsl/jbang/core/commands/AskLoopTest.java | 157 +++++++++++++++++++++
.../dsl/jbang/core/commands/FakeLlmServer.java | 22 ++-
5 files changed, 204 insertions(+), 20 deletions(-)
diff --git
a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
index cfd8900b0488..945722646904 100644
--- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
+++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
@@ -506,6 +506,13 @@ or follows, like Enter does.
* `camel ask` with an OpenAI-compatible server that gives the arguments of a
tool call as a JSON object, instead of JSON
text, calls the tool with these arguments (it called it without arguments).
+=== camel-jbang - camel ask: the exit code and the output of a question
+
+* `camel ask` exits with `1` when the call of the LLM fails, such as with an
error of the model or a refused connection:
+ it exited with `0` without an answer.
+* `camel ask` with a question prints only the answer on stdout. The model it
uses, the integration it asks about, the
+ note about a small model, and the output of `--show-tools` and
`--show-stats` go to stderr.
+
=== camel-jbang - exit codes and errors of more commands
* `camel export` does not clean the export directory when it is the current
directory, or one that contains it, unless
diff --git a/docs/user-manual/modules/ROOT/pages/camel-jbang-ai.adoc
b/docs/user-manual/modules/ROOT/pages/camel-jbang-ai.adoc
index c83a6760c54b..afd1da07aec9 100644
--- a/docs/user-manual/modules/ROOT/pages/camel-jbang-ai.adoc
+++ b/docs/user-manual/modules/ROOT/pages/camel-jbang-ai.adoc
@@ -51,6 +51,10 @@ Use `--show-tools` to see tool calls and results as they
happen:
camel ask "show me the route structure" --show-tools
----
+With a question, the answer is the only output on stdout, so a script can use
it: the model in use, the tool calls of
+`--show-tools` and the token usage of `--show-stats` are printed on stderr.
`camel ask` exits with `1` when the LLM
+cannot answer, such as when it is not reachable or the call fails.
+
=== What asks first
Most tools only read, and run without asking. A call that changes the project,
diff --git
a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Ask.java
b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Ask.java
index e02bff95386d..c5cd94ebcdd0 100644
---
a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Ask.java
+++
b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Ask.java
@@ -189,12 +189,12 @@ public class Ask extends CamelCommand {
}
String userQuestion = String.join(" ", question);
- printer().println("Using " + client.model + " (" + client.apiType + ",
" + (core ? "core" : "full")
- + " tools) to answer your question...");
+ // on stderr, so stdout has the answer only
+ printer().printDiagnostic("Using " + client.model + " (" +
client.apiType + ", " + (core ? "core" : "full")
+ + " tools) to answer your question...");
if (process != null) {
- printer().println("Target: " + process.name() + " (PID " +
process.pid() + ")");
+ printer().printDiagnostic("Target: " + process.name() + " (PID " +
process.pid() + ")");
}
- printer().println();
List<LlmClient.Message> messages = new ArrayList<>();
return runAgentLoop(client, systemPrompt, toolDefs, messages,
userQuestion);
@@ -252,7 +252,8 @@ public class Ask extends CamelCommand {
boolean corrected = false;
for (int i = 0; i < maxIterations; i++) {
LlmClient.ChatResponse response =
client.chatWithTools(systemPrompt, messages, tools);
- if (response == null) {
+ if (response == null || isFailed(response)) {
+ // the client printed why
printer().printErr("Failed to get response from LLM");
return 1;
}
@@ -264,11 +265,12 @@ public class Ask extends CamelCommand {
List<LlmClient.ToolResult> results = new ArrayList<>();
for (LlmClient.ToolCall toolCall : response.toolCalls()) {
if (showTools) {
- printer().println("[tool] " + toolCall.name() + "(" +
toolCall.arguments().toJson() + ")");
+ printer().printDiagnostic(
+ "[tool] " + toolCall.name() + "(" +
toolCall.arguments().toJson() + ")");
}
String result = askTools.executeTool(toolCall.name(),
toolCall.arguments());
if (showTools) {
- printer().println("[result] " + truncate(result, 200));
+ printer().printDiagnostic("[result] " +
truncate(result, 200));
}
results.add(new LlmClient.ToolResult(toolCall.id(),
result));
}
@@ -284,9 +286,8 @@ public class Ask extends CamelCommand {
corrected = true;
messages.add(LlmClient.Message.assistantWithToolCalls(response.text(),
List.of()));
messages.add(LlmClient.Message.user(AnswerChecks.correctionRequest(problems)));
- printer().println();
- printer().println("(the answer has an invalid simple
expression: asking for a corrected answer)");
- printer().println();
+ printer().printDiagnostic(
+ "(the answer has an invalid simple expression:
asking for a corrected answer)");
continue;
}
messages.add(LlmClient.Message.assistantWithToolCalls(response.text(),
List.of()));
@@ -313,8 +314,15 @@ public class Ask extends CamelCommand {
.append(LlmClient.formatTokens(usage.totalTokens())).append(" total tokens");
}
sb.append(")");
- printer().println();
- printer().println(sb.toString());
+ printer().printDiagnostic(sb.toString());
+ }
+
+ /**
+ * Whether the call of the LLM failed: the client answers with the error
stop reason, and nothing else.
+ */
+ static boolean isFailed(LlmClient.ChatResponse response) {
+ return "error".equals(response.stopReason()) && response.text() == null
+ && (response.toolCalls() == null ||
response.toolCalls().isEmpty());
}
// ---- Process discovery (delegates to RuntimeHelper) ----
@@ -364,7 +372,7 @@ public class Ask extends CamelCommand {
private void warnSmallModel(String name, double billions) {
String note = smallModelNote(name, billions);
if (note != null) {
- printer().println(note);
+ printer().printDiagnostic(note);
}
}
diff --git
a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/AskLoopTest.java
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/AskLoopTest.java
new file mode 100644
index 000000000000..7160966cbddf
--- /dev/null
+++
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/AskLoopTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.dsl.jbang.core.commands;
+
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import org.apache.camel.dsl.jbang.core.common.CommandLineHelper;
+import org.apache.camel.util.json.JsonArray;
+import org.apache.camel.util.json.JsonObject;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import picocli.CommandLine;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * camel ask with a question: the model answers, or calls tools until it
answers, against a local server that plays
+ * Ollama (no running integration). The answer is the only output on stdout.
+ */
+class AskLoopTest extends CamelCommandBaseTestSupport {
+
+ @TempDir
+ Path home;
+
+ private Path previousHome;
+ private FakeLlmServer server;
+
+ @BeforeEach
+ void start() throws Exception {
+ // no running integrations
+ previousHome = CommandLineHelper.getHomeDir();
+ CommandLineHelper.useHomeDir(home.toString());
+ server = new FakeLlmServer("unused");
+ }
+
+ @AfterEach
+ void stop() {
+ server.close();
+ CommandLineHelper.useHomeDir(previousHome.toString());
+ }
+
+ @Test
+ void theAnswerIsTheOutput() throws Exception {
+ server.reply("/api/chat", 200, answer("The routes are fine"));
+
+ assertThat(command().doCall()).isZero();
+
+ assertThat(printer.getStdout()).isEqualTo("The routes are fine");
+ assertThat(printer.getStderr()).isEqualTo("Using test (ollama, core
tools) to answer your question...");
+ JsonArray messages = (JsonArray) server.lastRequest().get("messages");
+ assertThat(((JsonObject)
messages.get(1)).getString("content")).isEqualTo("what routes are running?");
+ }
+
+ @Test
+ void theToolsTheModelCallsAreRunAndTheirResultsGiven() throws Exception {
+ server.reply("/api/chat", 200, toolCall("no_such_tool"),
answer("Done"));
+
+ assertThat(command("--show-tools").doCall()).isZero();
+
+ assertThat(printer.getStdout()).isEqualTo("Done");
+ assertThat(printer.getStderr()).contains("[tool]
no_such_tool({\"name\":\"orders\"})",
+ "[result] Unknown tool: no_such_tool");
+ // the second question has the call and its result
+ JsonArray messages = (JsonArray) server.lastRequest().get("messages");
+ JsonObject result = (JsonObject) messages.get(messages.size() - 1);
+ assertThat(result.getString("role")).isEqualTo("tool");
+ assertThat(result.getString("content")).isEqualTo("Unknown tool:
no_such_tool");
+ }
+
+ @Test
+ void aFailedCallOfTheLlmIsAFailure() throws Exception {
+ server.reply("/api/chat", 500, "{\"error\":\"model runner has
crashed\"}");
+
+ assertThat(command().doCall()).isEqualTo(1);
+
+ assertThat(printer.getStdout()).isEmpty();
+ assertThat(printer.getStderr()).contains("LLM returned status: 500",
"model runner has crashed",
+ "Failed to get response from LLM");
+ }
+
+ @Test
+ void noAnswerWithinTheMaximumIterationsIsAFailure() throws Exception {
+ server.reply("/api/chat", 200, toolCall("no_such_tool"));
+
+ assertThat(command("--max-iterations=2").doCall()).isEqualTo(1);
+
+ assertThat(printer.getStderr()).contains("Reached maximum iterations
(2) without a final answer.");
+ }
+
+ @Test
+ void theStatsOnStderr() throws Exception {
+ server.reply("/api/chat", 200, answer("The routes are fine"));
+
+ assertThat(command("--show-stats").doCall()).isZero();
+
+ assertThat(printer.getStdout()).isEqualTo("The routes are fine");
+ assertThat(printer.getStderr()).contains("10 input / 5 output / 15
total tokens");
+ }
+
+ private Ask command(String... args) {
+ Ask command = new Ask(new CamelJBangMain().withPrinter(printer));
+ CommandLine.populateCommand(command, Stream.concat(
+ Stream.of("what routes are running?", "--url=" + server.url(),
"--api-type=ollama", "--model=test"),
+ Arrays.stream(args)).toArray(String[]::new));
+ return command;
+ }
+
+ // as Ollama streams them: the answer in two parts, and the end with the
token counts
+ private static String answer(String text) {
+ int half = text.length() / 2;
+ return String.join("\n",
+ chunk(new JsonObject(Map.of("role", "assistant", "content",
text.substring(0, half))), false),
+ chunk(new JsonObject(Map.of("role", "assistant", "content",
text.substring(half))), false),
+ chunk(new JsonObject(Map.of("role", "assistant", "content",
"")), true), "");
+ }
+
+ private static String toolCall(String tool) {
+ JsonObject function = new JsonObject(Map.of("name", tool, "arguments",
new JsonObject(Map.of("name", "orders"))));
+ JsonObject message = new JsonObject(
+ Map.of("role", "assistant", "content", "",
+ "tool_calls", new JsonArray(List.of(new
JsonObject(Map.of("function", function))))));
+ return String.join("\n", chunk(message, false),
+ chunk(new JsonObject(Map.of("role", "assistant", "content",
"")), true), "");
+ }
+
+ private static String chunk(JsonObject message, boolean done) {
+ JsonObject chunk = new JsonObject();
+ chunk.put("message", message);
+ chunk.put("done", done);
+ if (done) {
+ chunk.put("done_reason", "stop");
+ chunk.put("prompt_eval_count", 10);
+ chunk.put("eval_count", 5);
+ }
+ return chunk.toJson();
+ }
+}
diff --git
a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/FakeLlmServer.java
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/FakeLlmServer.java
index b6f8599cfc21..c78115b7a910 100644
---
a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/FakeLlmServer.java
+++
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/FakeLlmServer.java
@@ -20,9 +20,11 @@ import java.io.IOException;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
+import java.util.Deque;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedDeque;
import com.sun.net.httpserver.Headers;
import com.sun.net.httpserver.HttpExchange;
@@ -40,8 +42,8 @@ final class FakeLlmServer implements AutoCloseable {
private final HttpServer server;
private final String answer;
- // the replies a test gave, by path
- private final Map<String, Reply> replies = new ConcurrentHashMap<>();
+ // the replies a test gave, by path: one for each request, and the last
one for the rest
+ private final Map<String, Deque<Reply>> replies = new
ConcurrentHashMap<>();
private volatile JsonObject lastRequest;
private volatile Headers lastHeaders;
@@ -60,10 +62,15 @@ final class FakeLlmServer implements AutoCloseable {
}
/**
- * Replies to the requests to the path with the status and body, instead
of the reply of the server.
+ * Replies to the requests to the path with the status and a body for each
request, instead of the reply of the
+ * server: the first request gets the first body, and the last body
answers the requests after it.
*/
- FakeLlmServer reply(String path, int status, String body) {
- replies.put(path, new Reply(status, body));
+ FakeLlmServer reply(String path, int status, String... bodies) {
+ Deque<Reply> list = new ConcurrentLinkedDeque<>();
+ for (String body : bodies) {
+ list.add(new Reply(status, body));
+ }
+ replies.put(path, list);
return this;
}
@@ -88,10 +95,11 @@ final class FakeLlmServer implements AutoCloseable {
private void handle(HttpExchange exchange) throws IOException {
String path = exchange.getRequestURI().getPath();
- Reply given = replies.get(path);
+ Deque<Reply> given = replies.get(path);
if (given != null) {
read(exchange);
- reply(exchange, given.status(), given.body());
+ Reply next = given.size() > 1 ? given.poll() : given.peek();
+ reply(exchange, next.status(), next.body());
return;
}
switch (path) {
