This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch fix/CAMEL-23688-llm-chat in repository https://gitbox.apache.org/repos/asf/camel.git
commit 81f5649cac24466e9615eb73faf894f8706675d4 Author: Claus Ibsen <[email protected]> AuthorDate: Fri Oct 16 23:08:49 2026 +0200 CAMEL-23688: camel-jbang - Tests for the chat with tools of the LLM client Tests for the chat with tools of LlmClient (38% covered), which camel ask uses, against FakeLlmServer playing each provider: the request in the format of Ollama, OpenAI-compatible servers and Anthropic (the system prompt, the tools, the tool calls and their results in the conversation), the answer with its tool calls, reasoning and token usage, streamed or not, and the errors. FakeLlmServer can now give the reply of any path, and keeps the headers of the last request. They found these bugs, fixed here: * A streamed answer that failed printed the error on stdout, after the answer so far, while the other errors go to stderr; camel explain, camel ask and camel overview --ai stream against Ollama by default. * A failure without a message, such as a refused connection, printed "null" instead of the message of its cause. * The arguments of a tool call that an OpenAI-compatible server gives as a JSON object, instead of JSON text, were dropped. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../ROOT/pages/camel-4x-upgrade-guide-4_24.adoc | 8 + .../camel/dsl/jbang/core/commands/Harden.java | 6 +- .../camel/dsl/jbang/core/commands/LlmClient.java | 59 +++- .../dsl/jbang/core/commands/FakeLlmServer.java | 63 +++- .../dsl/jbang/core/commands/LlmClientChatTest.java | 338 +++++++++++++++++++++ 5 files changed, 444 insertions(+), 30 deletions(-) diff --git a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc index b332441dfd88..0fd8984bed26 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc @@ -485,6 +485,14 @@ or follows, like Enter does. answered at once), and `camel_control stop-routes` and `start-routes` answer with what was done instead of the answer of the integration. +=== camel-jbang - the errors of a streamed answer of an LLM + +* When the streamed answer of an LLM fails, as with `camel explain`, `camel ask` or `camel overview --ai` against + Ollama, the error is printed on stderr: it was printed on stdout after the answer so far. A failure without a message + of its own, such as a refused connection, has the message of its cause instead of `null`. +* `camel ask` with an OpenAI-compatible server that gives the arguments of a tool call as a JSON object, instead of JSON + text, calls the tool with these arguments (it called it without arguments). + === camel-jbang - exit codes and errors of more commands * `camel export` does not clean the export directory when it is the current directory, or one that contains it, unless diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Harden.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Harden.java index 43d38db64081..2ad81ad4358b 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Harden.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Harden.java @@ -658,10 +658,10 @@ public class Harden extends CamelCommand { return fullResponse.toString(); } catch (HttpTimeoutException e) { - printer().printErr("\nRequest timed out after " + timeout + " seconds."); + printer().printErr("Request timed out after " + timeout + " seconds."); return null; } catch (Exception e) { - printer().printErr("\nError during streaming: " + e.getMessage()); + printer().printErr("Error during streaming: " + LlmClient.failure(e)); return null; } } @@ -691,7 +691,7 @@ public class Harden extends CamelCommand { printer().printErr("Request timed out after " + timeout + " seconds."); return null; } catch (Exception e) { - printer().printErr("Error calling LLM: " + e.getMessage()); + printer().printErr("Error calling LLM: " + LlmClient.failure(e)); return null; } } diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/LlmClient.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/LlmClient.java index 0f2e8679d168..c2485f5d505d 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/LlmClient.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/LlmClient.java @@ -1330,6 +1330,7 @@ public class LlmClient { } private ChatResponse streamOllamaChatRequest(String requestUrl, JsonObject body) { + StringBuilder fullText = new StringBuilder(); try { String jsonBody = body.toJson(); if (verbose) { @@ -1351,7 +1352,6 @@ public class LlmClient { return new ChatResponse(null, List.of(), "error", false, TokenUsage.EMPTY); } - StringBuilder fullText = new StringBuilder(); List<ToolCall> toolCalls = new ArrayList<>(); String[] doneReasonHolder = { null }; int[] tokenHolder = { 0, 0, 0 }; @@ -1436,14 +1436,37 @@ public class LlmClient { } return new ChatResponse(text, toolCalls, stopReason, true, usage); } catch (HttpTimeoutException e) { - printer.println("\nRequest timed out after " + timeout + " seconds."); + streamFailed(fullText, "Request timed out after " + timeout + " seconds."); return new ChatResponse(null, List.of(), "error", false, TokenUsage.EMPTY); } catch (Exception e) { - printer.println("\nError during streaming: " + e.getMessage()); + streamFailed(fullText, "Calling the LLM failed: " + failure(e)); return new ChatResponse(null, List.of(), "error", false, TokenUsage.EMPTY); } } + /** + * A streamed answer that failed: the end of the line of the answer printed so far, and the error on stderr, so the + * output has only the answer. + */ + private void streamFailed(CharSequence printed, String error) { + if (!printed.isEmpty()) { + printer.println(); + } + printer.printErr(error); + } + + /** + * The message of a failure, from its cause when it has none (a refused connection has no message of its own). + */ + static String failure(Exception e) { + for (Throwable t = e; t != null; t = t.getCause()) { + if (t.getMessage() != null && !t.getMessage().isBlank()) { + return t.getMessage(); + } + } + return e.getClass().getSimpleName(); + } + // ---- Shared message/tool builders ---- private JsonArray buildChatMessages(String systemPrompt, List<Message> messages, boolean ollamaNative) { @@ -1711,11 +1734,17 @@ public class LlmClient { JsonObject function = (JsonObject) tc.get("function"); String id = tc.getString("id"); String name = function.getString("name"); + // the arguments are JSON text, or an object with some OpenAI-compatible servers (as with Ollama) JsonObject args; - try { - args = (JsonObject) Jsoner.deserialize(function.getString("arguments")); - } catch (Exception e) { - args = new JsonObject(); + Object argsObj = function.get("arguments"); + if (argsObj instanceof JsonObject jo) { + args = jo; + } else { + try { + args = (JsonObject) Jsoner.deserialize((String) argsObj); + } catch (Exception e) { + args = new JsonObject(); + } } toolCalls.add(new ToolCall(id, name, args)); } @@ -1932,7 +1961,7 @@ public class LlmClient { printer.printErr("Request timed out after " + timeout + " seconds."); return null; } catch (Exception e) { - printer.printErr("Calling the LLM failed: " + e.getMessage()); + printer.printErr("Calling the LLM failed: " + failure(e)); return null; } } @@ -1970,12 +1999,13 @@ public class LlmClient { printer.printErr("Request timed out after " + timeout + " seconds."); return null; } catch (Exception e) { - printer.printErr("Calling the LLM failed: " + e.getMessage()); + printer.printErr("Calling the LLM failed: " + failure(e)); return null; } } String sendStreamingRequest(String requestUrl, JsonObject body, String authKey, String textField) { + StringBuilder fullResponse = new StringBuilder(); try { HttpRequest.Builder builder = HttpRequest.newBuilder() .uri(URI.create(requestUrl)) @@ -1996,7 +2026,6 @@ public class LlmClient { return null; } - StringBuilder fullResponse = new StringBuilder(); response.body().forEach(line -> { if (line.isBlank()) { return; @@ -2015,16 +2044,17 @@ public class LlmClient { printer.println(); return fullResponse.toString(); } catch (HttpTimeoutException e) { - printer.println("\nRequest timed out after " + timeout + " seconds."); + streamFailed(fullResponse, "Request timed out after " + timeout + " seconds."); return null; } catch (Exception e) { - printer.println("\nError during streaming: " + e.getMessage()); + streamFailed(fullResponse, "Calling the LLM failed: " + failure(e)); return null; } } private String sendAnthropicStreamingRequest(String requestUrl, JsonObject body, Map<String, String> headers) { body.put("stream", true); + StringBuilder fullResponse = new StringBuilder(); try { HttpRequest.Builder builder = HttpRequest.newBuilder() .uri(URI.create(requestUrl)) @@ -2044,7 +2074,6 @@ public class LlmClient { return null; } - StringBuilder fullResponse = new StringBuilder(); response.body().forEach(line -> { if (line.isBlank() || !line.startsWith("data: ")) { return; @@ -2073,10 +2102,10 @@ public class LlmClient { printer.println(); return fullResponse.toString(); } catch (HttpTimeoutException e) { - printer.println("\nRequest timed out after " + timeout + " seconds."); + streamFailed(fullResponse, "Request timed out after " + timeout + " seconds."); return null; } catch (Exception e) { - printer.println("\nError during streaming: " + e.getMessage()); + streamFailed(fullResponse, "Calling the LLM failed: " + failure(e)); return null; } } diff --git a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/FakeLlmServer.java b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/FakeLlmServer.java index e60017e02118..b6f8599cfc21 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/FakeLlmServer.java +++ b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/FakeLlmServer.java @@ -22,7 +22,9 @@ import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import com.sun.net.httpserver.Headers; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpServer; import org.apache.camel.util.json.JsonArray; @@ -31,23 +33,25 @@ import org.apache.camel.util.json.Jsoner; /** * A local server that plays an LLM for the tests of the AI commands: Ollama (its models, and the answer of the model, - * whole or streamed in two parts) and an OpenAI-compatible chat completion. It keeps the last question it answered. + * whole or streamed in two parts) and an OpenAI-compatible chat completion. A test can give the reply of a path + * instead, such as a chat with tool calls or an error. It keeps the last question it answered, and its headers. */ final class FakeLlmServer implements AutoCloseable { private final HttpServer server; private final String answer; + // the replies a test gave, by path + private final Map<String, Reply> replies = new ConcurrentHashMap<>(); private volatile JsonObject lastRequest; + private volatile Headers lastHeaders; + + private record Reply(int status, String body) { + } FakeLlmServer(String answer) throws IOException { this.answer = answer; server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); - server.createContext("/api/tags", ex -> reply(ex, "{\"models\":[{\"name\":\"test\",\"size\":1000}]}")); - server.createContext("/api/ps", ex -> reply(ex, "{\"models\":[]}")); - server.createContext("/api/show", ex -> reply(ex, "{\"model_info\":{}}")); - server.createContext("/api/generate", this::generate); - server.createContext("/v1/models", ex -> reply(ex, "{\"data\":[{\"id\":\"test\"}]}")); - server.createContext("/v1/chat/completions", this::chat); + server.createContext("/", this::handle); server.start(); } @@ -55,6 +59,14 @@ final class FakeLlmServer implements AutoCloseable { return "http://127.0.0.1:"; + server.getAddress().getPort(); } + /** + * Replies to the requests to the path with the status and body, instead of the reply of the server. + */ + FakeLlmServer reply(String path, int status, String body) { + replies.put(path, new Reply(status, body)); + return this; + } + /** * The last question the server answered, as the client sent it. */ @@ -62,11 +74,37 @@ final class FakeLlmServer implements AutoCloseable { return lastRequest; } + /** + * The headers of the last question. + */ + Headers lastHeaders() { + return lastHeaders; + } + @Override public void close() { server.stop(0); } + private void handle(HttpExchange exchange) throws IOException { + String path = exchange.getRequestURI().getPath(); + Reply given = replies.get(path); + if (given != null) { + read(exchange); + reply(exchange, given.status(), given.body()); + return; + } + switch (path) { + case "/api/tags" -> reply(exchange, 200, "{\"models\":[{\"name\":\"test\",\"size\":1000}]}"); + case "/api/ps" -> reply(exchange, 200, "{\"models\":[]}"); + case "/api/show" -> reply(exchange, 200, "{\"model_info\":{}}"); + case "/api/generate" -> generate(exchange); + case "/v1/models" -> reply(exchange, 200, "{\"data\":[{\"id\":\"test\"}]}"); + case "/v1/chat/completions" -> chat(exchange); + default -> reply(exchange, 404, ""); + } + } + private void generate(HttpExchange exchange) throws IOException { JsonObject request = read(exchange); if (request.getBooleanOrDefault("stream", false)) { @@ -74,9 +112,9 @@ final class FakeLlmServer implements AutoCloseable { int half = answer.length() / 2; String first = new JsonObject(Map.of("response", answer.substring(0, half), "done", false)).toJson(); String last = new JsonObject(Map.of("response", answer.substring(half), "done", true)).toJson(); - reply(exchange, first + "\n" + last + "\n"); + reply(exchange, 200, first + "\n" + last + "\n"); } else { - reply(exchange, new JsonObject(Map.of("response", answer, "done", true)).toJson()); + reply(exchange, 200, new JsonObject(Map.of("response", answer, "done", true)).toJson()); } } @@ -84,20 +122,21 @@ final class FakeLlmServer implements AutoCloseable { read(exchange); JsonObject message = new JsonObject(Map.of("role", "assistant", "content", answer)); JsonObject choice = new JsonObject(Map.of("index", 0, "message", message)); - reply(exchange, new JsonObject(Map.of("choices", new JsonArray(List.of(choice)))).toJson()); + reply(exchange, 200, new JsonObject(Map.of("choices", new JsonArray(List.of(choice)))).toJson()); } private JsonObject read(HttpExchange exchange) throws IOException { String body = new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8); JsonObject request = Jsoner.deserialize(body, new JsonObject()); lastRequest = request; + lastHeaders = exchange.getRequestHeaders(); return request; } - private static void reply(HttpExchange exchange, String body) throws IOException { + private static void reply(HttpExchange exchange, int status, String body) throws IOException { byte[] bytes = body.getBytes(StandardCharsets.UTF_8); exchange.getResponseHeaders().add("Content-Type", "application/json"); - exchange.sendResponseHeaders(200, bytes.length); + exchange.sendResponseHeaders(status, bytes.length == 0 ? -1 : bytes.length); try (OutputStream out = exchange.getResponseBody()) { out.write(bytes); } diff --git a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/LlmClientChatTest.java b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/LlmClientChatTest.java new file mode 100644 index 000000000000..081eb955f218 --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/LlmClientChatTest.java @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands; + +import java.io.IOException; +import java.net.ServerSocket; +import java.util.List; +import java.util.Map; + +import org.apache.camel.dsl.jbang.core.commands.LlmClient.ApiType; +import org.apache.camel.dsl.jbang.core.commands.LlmClient.ChatResponse; +import org.apache.camel.dsl.jbang.core.commands.LlmClient.Message; +import org.apache.camel.dsl.jbang.core.commands.LlmClient.ToolCall; +import org.apache.camel.dsl.jbang.core.commands.LlmClient.ToolDef; +import org.apache.camel.dsl.jbang.core.commands.LlmClient.ToolResult; +import org.apache.camel.dsl.jbang.core.common.StringPrinter; +import org.apache.camel.util.json.JsonArray; +import org.apache.camel.util.json.JsonObject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The chat with tools of camel ask, against a local server that plays each provider: the request in the format of the + * provider (Ollama, OpenAI-compatible, Anthropic), the answer with its tool calls and token usage, and the errors. + */ +class LlmClientChatTest { + + private static final List<ToolDef> TOOLS = List.of(new ToolDef( + "get_routes", "The routes of the integration", + new JsonObject(Map.of("type", "object", "properties", new JsonObject())))); + + private final StringPrinter printer = new StringPrinter(); + private FakeLlmServer server; + + @BeforeEach + void start() throws IOException { + server = new FakeLlmServer("unused"); + } + + @AfterEach + void stop() { + server.close(); + } + + @Test + void ollamaChatWithToolCalls() { + server.reply("/api/chat", 200, """ + {"message":{"role":"assistant","content":"Let me look", + "tool_calls":[{"function":{"name":"get_routes","arguments":{"name":"orders"}}}]}, + "done":true,"done_reason":"stop","prompt_eval_count":100,"eval_count":20, + "prompt_eval_cached_count":80,"eval_duration":2000000000}"""); + + ChatResponse response = client(ApiType.ollama).chatWithTools("You help", List.of(Message.user("Hi")), TOOLS); + + assertThat(response.text()).isEqualTo("Let me look"); + assertThat(response.stopReason()).isEqualTo("tool_calls"); + assertThat(response.streamed()).isFalse(); + ToolCall call = response.toolCalls().get(0); + assertThat(call.id()).isEqualTo("call_0"); + assertThat(call.name()).isEqualTo("get_routes"); + assertThat(call.arguments().getString("name")).isEqualTo("orders"); + assertThat(response.usage().inputTokens()).isEqualTo(100); + assertThat(response.usage().outputTokens()).isEqualTo(20); + assertThat(response.usage().totalTokens()).isEqualTo(120); + assertThat(response.usage().cachedTokens()).isEqualTo(80); + assertThat(response.usage().generationMillis()).isEqualTo(2000); + + JsonObject request = server.lastRequest(); + assertThat(request.getString("model")).isEqualTo("test"); + assertThat(request.getBoolean("stream")).isFalse(); + assertThat(request.getBoolean("think")).isFalse(); + assertThat(request.getString("keep_alive")).isEqualTo("30m"); + JsonObject options = request.getMap("options"); + assertThat(options.getInteger("num_ctx")).isPositive(); + JsonArray tools = (JsonArray) request.get("tools"); + JsonObject function = ((JsonObject) tools.get(0)).getMap("function"); + assertThat(function.getString("name")).isEqualTo("get_routes"); + } + + @Test + void ollamaChatStreamed() { + server.reply("/api/chat", 200, String.join("\n", + "{\"message\":{\"role\":\"assistant\",\"content\":\"The routes \"},\"done\":false}", + "{\"message\":{\"role\":\"assistant\",\"content\":\"are fine\"},\"done\":false}", + "{\"message\":{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"function\":" + + "{\"name\":\"get_routes\",\"arguments\":\"{\\\"name\\\":\\\"orders\\\"}\"}}]}," + + "\"done\":false}", + "{\"message\":{\"role\":\"assistant\",\"content\":\"\"},\"done\":true,\"done_reason\":\"stop\"," + + "\"prompt_eval_count\":10,\"eval_count\":5}", + "")); + + ChatResponse response = client(ApiType.ollama).withStream(true) + .chatWithTools("You help", List.of(Message.user("Hi")), TOOLS); + + // the answer is printed as it comes + assertThat(printer.getStdout()).isEqualTo("The routes are fine"); + assertThat(response.text()).isEqualTo("The routes are fine"); + assertThat(response.streamed()).isTrue(); + assertThat(response.stopReason()).isEqualTo("tool_calls"); + // the arguments as text are read as JSON + assertThat(response.toolCalls().get(0).arguments().getString("name")).isEqualTo("orders"); + assertThat(response.usage().totalTokens()).isEqualTo(15); + assertThat(server.lastRequest().getBoolean("stream")).isTrue(); + } + + @Test + void openAiChatWithToolCalls() { + server.reply("/v1/chat/completions", 200, """ + {"choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","content":null, + "tool_calls":[{"id":"call_abc","type":"function", + "function":{"name":"get_routes","arguments":"{\\"name\\":\\"orders\\"}"}}]}}], + "usage":{"prompt_tokens":50,"completion_tokens":10,"total_tokens":60, + "prompt_tokens_details":{"cached_tokens":40}}}"""); + + ChatResponse response = client(ApiType.openai).withApiKey("sk-secret") + .chatWithTools("You help", List.of(Message.user("Hi")), TOOLS); + + assertThat(response.stopReason()).isEqualTo("tool_calls"); + ToolCall call = response.toolCalls().get(0); + assertThat(call.id()).isEqualTo("call_abc"); + assertThat(call.arguments().getString("name")).isEqualTo("orders"); + assertThat(response.usage().totalTokens()).isEqualTo(60); + assertThat(response.usage().cachedTokens()).isEqualTo(40); + assertThat(server.lastHeaders().getFirst("Authorization")).isEqualTo("Bearer sk-secret"); + assertThat(server.lastRequest().getString("model")).isEqualTo("test"); + } + + @Test + void openAiToolCallArgumentsAsAnObject() { + // some OpenAI-compatible servers send the arguments as an object, as Ollama does + server.reply("/v1/chat/completions", 200, """ + {"choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant", + "tool_calls":[{"id":"call_abc","type":"function", + "function":{"name":"get_routes","arguments":{"name":"orders"}}}]}}]}"""); + + ChatResponse response = client(ApiType.openai).chatWithTools("You help", List.of(Message.user("Hi")), TOOLS); + + assertThat(response.toolCalls().get(0).arguments().getString("name")).isEqualTo("orders"); + } + + @Test + void openAiReasoningWhenThereIsNoContent() { + server.reply("/v1/chat/completions", 200, """ + {"choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","content":"", + "reasoning_content":"The routes are fine"}}]}"""); + + ChatResponse response = client(ApiType.openai).chatWithTools("You help", List.of(Message.user("Hi")), TOOLS); + + assertThat(response.text()).isEqualTo("The routes are fine"); + assertThat(response.stopReason()).isEqualTo("stop"); + } + + @Test + void anthropicChatWithToolUse() { + server.reply("/v1/messages", 200, """ + {"stop_reason":"tool_use","content":[{"type":"text","text":"Let me look"}, + {"type":"tool_use","id":"toolu_1","name":"get_routes","input":{"name":"orders"}}], + "usage":{"input_tokens":30,"output_tokens":7,"cache_read_input_tokens":20}}"""); + + ChatResponse response = client(ApiType.anthropic).withApiKey("sk-ant") + .chatWithTools("You help", List.of(Message.user("Hi")), TOOLS); + + assertThat(response.text()).isEqualTo("Let me look"); + assertThat(response.stopReason()).isEqualTo("tool_use"); + ToolCall call = response.toolCalls().get(0); + assertThat(call.id()).isEqualTo("toolu_1"); + assertThat(call.arguments().getString("name")).isEqualTo("orders"); + assertThat(response.usage().totalTokens()).isEqualTo(37); + assertThat(response.usage().cachedTokens()).isEqualTo(20); + + assertThat(server.lastHeaders().getFirst("x-api-key")).isEqualTo("sk-ant"); + assertThat(server.lastHeaders().getFirst("anthropic-version")).isEqualTo("2023-06-01"); + JsonObject request = server.lastRequest(); + // the system prompt apart from the messages, and the tools with their input schema + assertThat(request.getString("system")).isEqualTo("You help"); + assertThat(request.getInteger("max_tokens")).isEqualTo(4096); + JsonObject tool = (JsonObject) ((JsonArray) request.get("tools")).get(0); + assertThat(tool.getString("name")).isEqualTo("get_routes"); + JsonObject schema = tool.getMap("input_schema"); + assertThat(schema).containsEntry("type", "object"); + } + + @Test + void theConversationInTheFormatOfOpenAi() { + server.reply("/v1/chat/completions", 200, "{\"choices\":[{\"message\":{\"content\":\"Done\"}}]}"); + + client(ApiType.openai).chatWithTools("You help", conversation(), TOOLS); + + JsonArray messages = (JsonArray) server.lastRequest().get("messages"); + assertThat(messages).hasSize(4); + assertThat(((JsonObject) messages.get(0)).getString("role")).isEqualTo("system"); + JsonObject call = (JsonObject) ((JsonArray) ((JsonObject) messages.get(2)).get("tool_calls")).get(0); + assertThat(call.getString("id")).isEqualTo("call_1"); + assertThat(call.getString("type")).isEqualTo("function"); + // the arguments as text + JsonObject function = call.getMap("function"); + assertThat(function.getString("arguments")).isEqualTo("{\"name\":\"orders\"}"); + JsonObject result = (JsonObject) messages.get(3); + assertThat(result.getString("role")).isEqualTo("tool"); + assertThat(result.getString("tool_call_id")).isEqualTo("call_1"); + assertThat(result.getString("content")).isEqualTo("orders: Started"); + } + + @Test + void theConversationInTheFormatOfOllama() { + server.reply("/api/chat", 200, "{\"message\":{\"content\":\"Done\"},\"done\":true}"); + + client(ApiType.ollama).chatWithTools("You help", conversation(), TOOLS); + + JsonArray messages = (JsonArray) server.lastRequest().get("messages"); + JsonObject call = (JsonObject) ((JsonArray) ((JsonObject) messages.get(2)).get("tool_calls")).get(0); + assertThat(call.containsKey("id")).isFalse(); + // the arguments as an object + JsonObject function = call.getMap("function"); + JsonObject arguments = function.getMap("arguments"); + assertThat(arguments).containsEntry("name", "orders"); + JsonObject result = (JsonObject) messages.get(3); + assertThat(result.containsKey("tool_call_id")).isFalse(); + assertThat(result.getString("content")).isEqualTo("orders: Started"); + } + + @Test + void theConversationInTheFormatOfAnthropic() { + server.reply("/v1/messages", 200, "{\"stop_reason\":\"end_turn\",\"content\":[{\"type\":\"text\",\"text\":\"Done\"}]}"); + + client(ApiType.anthropic).chatWithTools("You help", conversation(), TOOLS); + + JsonArray messages = (JsonArray) server.lastRequest().get("messages"); + // no system message: it is apart + assertThat(messages).hasSize(3); + JsonArray assistant = (JsonArray) ((JsonObject) messages.get(1)).get("content"); + JsonObject toolUse = (JsonObject) assistant.get(1); + assertThat(toolUse.getString("type")).isEqualTo("tool_use"); + assertThat(toolUse.getString("id")).isEqualTo("call_1"); + JsonObject input = toolUse.getMap("input"); + assertThat(input).containsEntry("name", "orders"); + JsonObject user = (JsonObject) messages.get(2); + assertThat(user.getString("role")).isEqualTo("user"); + JsonObject toolResult = (JsonObject) ((JsonArray) user.get("content")).get(0); + assertThat(toolResult.getString("type")).isEqualTo("tool_result"); + assertThat(toolResult.getString("tool_use_id")).isEqualTo("call_1"); + } + + @Test + void anErrorStatusIsReportedOnStderr() { + server.reply("/v1/chat/completions", 401, "{\"error\":{\"message\":\"Incorrect API key provided\"}}"); + + ChatResponse response = client(ApiType.openai).chatWithTools("You help", List.of(Message.user("Hi")), TOOLS); + + assertThat(response.stopReason()).isEqualTo("error"); + assertThat(response.text()).isNull(); + assertThat(printer.getStdout()).isEmpty(); + assertThat(printer.getStderr()).contains("LLM returned status: 401", "Authentication failed. Check your API key.", + "Incorrect API key provided"); + } + + @Test + void aStreamedErrorStatusIsReportedOnStderr() { + server.reply("/api/chat", 404, "{\"error\":\"model 'test' not found\"}"); + + ChatResponse response = client(ApiType.ollama).withStream(true) + .chatWithTools("You help", List.of(Message.user("Hi")), TOOLS); + + assertThat(response.stopReason()).isEqualTo("error"); + assertThat(printer.getStdout()).isEmpty(); + assertThat(printer.getStderr()).contains("LLM returned status: 404", "Endpoint not found.", + "model 'test' not found"); + } + + @Test + void aStreamThatCannotConnectIsReportedOnStderr() throws IOException { + String closed; + try (ServerSocket socket = new ServerSocket(0)) { + closed = "http://127.0.0.1:"; + socket.getLocalPort(); + } + + ChatResponse response = client(ApiType.ollama).withUrl(closed).withStream(true) + .chatWithTools("You help", List.of(Message.user("Hi")), TOOLS); + + assertThat(response.stopReason()).isEqualTo("error"); + // nothing that looks like an answer + assertThat(printer.getStdout()).isEmpty(); + assertThat(printer.getStderr()).startsWith("ERROR: Calling the LLM failed: ").doesNotContain("null"); + } + + @Test + void aGenerateStreamThatCannotConnectIsReportedOnStderr() throws IOException { + String closed; + try (ServerSocket socket = new ServerSocket(0)) { + closed = "http://127.0.0.1:"; + socket.getLocalPort(); + } + + String answer = client(ApiType.ollama).withUrl(closed).withStream(true).generate("You help", "Hi"); + + assertThat(answer).isNull(); + assertThat(printer.getStdout()).isEmpty(); + assertThat(printer.getStderr()).startsWith("ERROR: Calling the LLM failed: ").doesNotContain("null"); + } + + @Test + void theMessageOfAnError() { + assertThat(LlmClient.extractErrorMessage("{\"error\":{\"message\":\"Bad model\"}}")).isEqualTo("Bad model"); + assertThat(LlmClient.extractErrorMessage("{\"error\":\"model not found\"}")).isEqualTo("model not found"); + assertThat(LlmClient.extractErrorMessage("{\"detail\":\"Too many\"}")).isEqualTo("{\"detail\":\"Too many\"}"); + // an HTML page is not shown + assertThat(LlmClient.extractErrorMessage("<html><body>Bad Gateway</body></html>")).isNull(); + } + + private LlmClient client(ApiType apiType) { + return LlmClient.create().withApiType(apiType).withUrl(server.url()).withModel("test").withTimeout(10) + .withPrinter(printer); + } + + // the question, the tool the model called, and its result + private static List<Message> conversation() { + ToolCall call = new ToolCall("call_1", "get_routes", new JsonObject(Map.of("name", "orders"))); + return List.of(Message.user("How are the routes?"), + Message.assistantWithToolCalls("Let me look", List.of(call)), + Message.toolResults(List.of(new ToolResult("call_1", "orders: Started")))); + } +}
