This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new ec354b868506 CAMEL-26424: camel-jbang - turn thinking off with local
models in camel ask, explain and overview (#28742)
ec354b868506 is described below
commit ec354b86850646a600049a131102813dd9bc007d
Author: Claus Ibsen <[email protected]>
AuthorDate: Sat Oct 31 18:12:33 2026 +0100
CAMEL-26424: camel-jbang - turn thinking off with local models in camel
ask, explain and overview (#28742)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../camel/dsl/jbang/core/commands/LlmClient.java | 58 ++++++++++++
.../dsl/jbang/core/commands/FakeLlmServer.java | 21 ++++-
.../dsl/jbang/core/commands/LlmClientChatTest.java | 101 +++++++++++++++++++++
3 files changed, 179 insertions(+), 1 deletion(-)
diff --git
a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/LlmClient.java
b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/LlmClient.java
index 07aef3d6b984..11d280f58737 100644
---
a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/LlmClient.java
+++
b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/LlmClient.java
@@ -648,6 +648,7 @@ public class LlmClient {
request.put("model", model);
request.put("prompt", userPrompt);
request.put("system", systemPrompt);
+ request.put("think", false);
request.put("stream", stream);
request.put("keep_alive", OLLAMA_KEEP_ALIVE);
request.put("options", ollamaOptions());
@@ -670,6 +671,7 @@ public class LlmClient {
request.put("model", model);
request.put("messages", messages);
request.put("temperature", temperature);
+ turnThinkingOff(request);
String resolvedKey = resolveApiKey();
String apiUrl = normalizeOpenAiUrl(url);
@@ -1016,6 +1018,7 @@ public class LlmClient {
if (jsonTools != null) {
request.put("tools", jsonTools);
}
+ turnThinkingOff(request);
String apiUrl = normalizeOpenAiUrl(url);
String resolvedKey = resolveApiKey();
@@ -1024,6 +1027,57 @@ public class LlmClient {
return parseOpenAiChatResponse(response);
}
+ // ---- Thinking off with a local OpenAI-compatible server ----
+
+ // the fields for thinking a server refused, left out from then on
+ private volatile boolean reasoningEffortRefused;
+ private volatile boolean templateKwargsRefused;
+
+ /**
+ * Turns thinking off in a request to an OpenAI-compatible server on this
machine, as {@code "think": false} does on
+ * Ollama's own API. Local servers (Ollama's /v1, llama.cpp server, vLLM,
LM Studio) render the model's own chat
+ * template, and the Qwen 3.x templates think by default, which makes
every answer many times slower. Ollama reads
+ * {@code reasoning_effort}, llama.cpp reads {@code chat_template_kwargs},
and vLLM reads both. Hosted endpoints get
+ * neither field, as they may refuse fields they do not know.
+ */
+ void turnThinkingOff(JsonObject request) {
+ if (apiType == ApiType.openai && isLocalEndpoint()) {
+ if (!reasoningEffortRefused) {
+ request.put("reasoning_effort", "none");
+ }
+ if (!templateKwargsRefused) {
+ request.put("chat_template_kwargs", new
JsonObject(Map.of("enable_thinking", false)));
+ }
+ }
+ }
+
+ /**
+ * Sends a refused request again with one field for thinking less: first
without {@code reasoning_effort}, as a
+ * server that checks its requests may not know the value none, then also
without {@code chat_template_kwargs}. The
+ * field whose removal made the request succeed is left out from now on.
When the request fails without both, its
+ * error is reported, as for any request.
+ */
+ private JsonObject sendWithFewerThinkingFields(String requestUrl,
JsonObject body, String authKey) {
+ JsonObject request = new JsonObject(body);
+ boolean withoutEffort = request.remove("reasoning_effort") != null;
+ if (!withoutEffort) {
+ request.remove("chat_template_kwargs");
+ }
+ JsonObject response = sendRequest(requestUrl, request, authKey);
+ if (response != null) {
+ if (withoutEffort) {
+ reasoningEffortRefused = true;
+ } else {
+ templateKwargsRefused = true;
+ }
+ if (verbose) {
+ printer.println("[verbose] The server refused " +
(withoutEffort ? "reasoning_effort" : "chat_template_kwargs")
+ + ": it is left out from now on");
+ }
+ }
+ return response;
+ }
+
private JsonObject ollamaOptions() {
JsonObject options = new JsonObject();
options.put("temperature", temperature);
@@ -1958,6 +2012,10 @@ public class LlmClient {
if (response.statusCode() == 200) {
return (JsonObject) Jsoner.deserialize(response.body());
}
+ if ((response.statusCode() == 400 || response.statusCode() == 422)
+ && (body.containsKey("reasoning_effort") ||
body.containsKey("chat_template_kwargs"))) {
+ return sendWithFewerThinkingFields(requestUrl, body, authKey);
+ }
handleErrorStatus(response.statusCode(), response.body());
return null;
diff --git
a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/FakeLlmServer.java
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/FakeLlmServer.java
index c78115b7a910..0c8c8dd3186d 100644
---
a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/FakeLlmServer.java
+++
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/FakeLlmServer.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.CopyOnWriteArrayList;
import com.sun.net.httpserver.Headers;
import com.sun.net.httpserver.HttpExchange;
@@ -36,7 +37,8 @@ import org.apache.camel.util.json.Jsoner;
/**
* A local server that plays an LLM for the tests of the AI commands: Ollama
(its models, and the answer of the model,
* whole or streamed in two parts) and an OpenAI-compatible chat completion. A
test can give the reply of a path
- * instead, such as a chat with tool calls or an error. It keeps the last
question it answered, and its headers.
+ * instead, such as a chat with tool calls or an error. It keeps the questions
it answered, and the headers of the last
+ * one.
*/
final class FakeLlmServer implements AutoCloseable {
@@ -44,6 +46,7 @@ final class FakeLlmServer implements AutoCloseable {
private final String answer;
// the replies a test gave, by path: one for each request, and the last
one for the rest
private final Map<String, Deque<Reply>> replies = new
ConcurrentHashMap<>();
+ private final List<JsonObject> requests = new CopyOnWriteArrayList<>();
private volatile JsonObject lastRequest;
private volatile Headers lastHeaders;
@@ -74,6 +77,14 @@ final class FakeLlmServer implements AutoCloseable {
return this;
}
+ /**
+ * Adds a reply with its own status after the replies given for the path,
such as an answer after an error.
+ */
+ FakeLlmServer thenReply(String path, int status, String body) {
+ replies.computeIfAbsent(path, p -> new
ConcurrentLinkedDeque<>()).add(new Reply(status, body));
+ return this;
+ }
+
/**
* The last question the server answered, as the client sent it.
*/
@@ -81,6 +92,13 @@ final class FakeLlmServer implements AutoCloseable {
return lastRequest;
}
+ /**
+ * The questions the server answered, in order, as the client sent them.
+ */
+ List<JsonObject> requests() {
+ return List.copyOf(requests);
+ }
+
/**
* The headers of the last question.
*/
@@ -136,6 +154,7 @@ final class FakeLlmServer implements AutoCloseable {
private JsonObject read(HttpExchange exchange) throws IOException {
String body = new String(exchange.getRequestBody().readAllBytes(),
StandardCharsets.UTF_8);
JsonObject request = Jsoner.deserialize(body, new JsonObject());
+ requests.add(request);
lastRequest = request;
lastHeaders = exchange.getRequestHeaders();
return request;
diff --git
a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/LlmClientChatTest.java
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/LlmClientChatTest.java
index 291bd6b5f9bb..24b6efcdd1c2 100644
---
a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/LlmClientChatTest.java
+++
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/LlmClientChatTest.java
@@ -167,6 +167,107 @@ class LlmClientChatTest {
assertThat(response.stopReason()).isEqualTo("stop");
}
+ @Test
+ void openAiChatWithALocalServerTurnsThinkingOff() {
+ server.reply("/v1/chat/completions", 200,
"{\"choices\":[{\"message\":{\"content\":\"Done\"}}]}");
+
+ client(ApiType.openai).chatWithTools("You help",
List.of(Message.user("Hi")), TOOLS);
+
+ // Ollama's /v1 reads reasoning_effort, llama.cpp reads
chat_template_kwargs, vLLM reads both
+ JsonObject request = server.lastRequest();
+ assertThat(request.getString("reasoning_effort")).isEqualTo("none");
+ assertThat(((JsonObject)
request.get("chat_template_kwargs")).getBoolean("enable_thinking")).isFalse();
+ }
+
+ @Test
+ void anExplanationWithALocalServerTurnsThinkingOff() {
+ server.reply("/v1/chat/completions", 200,
"{\"choices\":[{\"message\":{\"content\":\"Done\"}}]}");
+
+ String answer = client(ApiType.openai).generate("You explain", "The
route");
+
+ assertThat(answer).isEqualTo("Done");
+
assertThat(server.lastRequest().getString("reasoning_effort")).isEqualTo("none");
+ }
+
+ @Test
+ void anExplanationWithOllamaTurnsThinkingOff() {
+ server.reply("/api/generate", 200,
"{\"response\":\"Done\",\"done\":true}");
+
+ String answer = client(ApiType.ollama).generate("You explain", "The
route");
+
+ // as the chat does: the Qwen 3.x models think by default, which makes
an explanation many times slower
+ assertThat(answer).isEqualTo("Done");
+ assertThat(server.lastRequest().getBoolean("think")).isFalse();
+ }
+
+ @Test
+ void aHostedServerGetsNoFieldsForThinking() {
+ JsonObject request = new JsonObject();
+
+
LlmClient.create().withApiType(ApiType.openai).withUrl("https://api.openai.com/v1";).turnThinkingOff(request);
+
LlmClient.create().withApiType(ApiType.openai).withUrl("https://myresource.openai.azure.com";)
+ .turnThinkingOff(request);
+
LlmClient.create().withApiType(ApiType.watsonx).withUrl("https://us-south.ml.cloud.ibm.com";)
+ .turnThinkingOff(request);
+
+ assertThat(request).isEmpty();
+ }
+
+ @Test
+ void aServerThatRefusesReasoningEffortGetsTheQuestionAgainWithoutIt() {
+ server.reply("/v1/chat/completions", 400,
+ "{\"error\":{\"message\":\"reasoning_effort: Input should be
'low', 'medium' or 'high'\"}}")
+ .thenReply("/v1/chat/completions", 200,
"{\"choices\":[{\"message\":{\"content\":\"Done\"}}]}");
+ LlmClient client = client(ApiType.openai);
+
+ ChatResponse response = client.chatWithTools("You help",
List.of(Message.user("Hi")), TOOLS);
+ client.chatWithTools("You help", List.of(Message.user("Hi again")),
TOOLS);
+
+ assertThat(response.text()).isEqualTo("Done");
+ assertThat(printer.getStderr()).isEmpty();
+ List<JsonObject> requests = server.requests();
+ assertThat(requests).hasSize(3);
+ assertThat(requests.get(0)).containsKeys("reasoning_effort",
"chat_template_kwargs");
+ // the same question again without it, and the next question leaves it
out from the start
+
assertThat(requests.get(1)).doesNotContainKey("reasoning_effort").containsKey("chat_template_kwargs");
+
assertThat(requests.get(2)).doesNotContainKey("reasoning_effort").containsKey("chat_template_kwargs");
+ }
+
+ @Test
+ void aServerThatRefusesBothFieldsForThinkingGetsTheQuestionWithoutThem() {
+ server.reply("/v1/chat/completions", 422, "{\"detail\":\"Extra inputs
are not permitted\"}",
+ "{\"detail\":\"Extra inputs are not permitted\"}")
+ .thenReply("/v1/chat/completions", 200,
"{\"choices\":[{\"message\":{\"content\":\"Done\"}}]}");
+ LlmClient client = client(ApiType.openai);
+
+ ChatResponse response = client.chatWithTools("You help",
List.of(Message.user("Hi")), TOOLS);
+ client.chatWithTools("You help", List.of(Message.user("Hi again")),
TOOLS);
+
+ assertThat(response.text()).isEqualTo("Done");
+ assertThat(printer.getStderr()).isEmpty();
+ List<JsonObject> requests = server.requests();
+ assertThat(requests).hasSize(4);
+
assertThat(requests.get(1)).doesNotContainKey("reasoning_effort").containsKey("chat_template_kwargs");
+ assertThat(requests.get(2)).doesNotContainKeys("reasoning_effort",
"chat_template_kwargs");
+ assertThat(requests.get(3)).doesNotContainKeys("reasoning_effort",
"chat_template_kwargs");
+ }
+
+ @Test
+ void aRefusalForAnotherReasonIsReportedOnce() {
+ server.reply("/v1/chat/completions", 400,
"{\"error\":{\"message\":\"The prompt is too long\"}}");
+ LlmClient client = client(ApiType.openai);
+
+ ChatResponse response = client.chatWithTools("You help",
List.of(Message.user("Hi")), TOOLS);
+
+ assertThat(response.stopReason()).isEqualTo("error");
+ assertThat(printer.getStderr()).containsOnlyOnce("LLM returned status:
400").contains("The prompt is too long");
+ assertThat(server.requests()).hasSize(3);
+
+ // the fields were not the cause, so the next question has them again
+ client.chatWithTools("You help", List.of(Message.user("Hi again")),
TOOLS);
+ assertThat(server.requests().get(3)).containsKeys("reasoning_effort",
"chat_template_kwargs");
+ }
+
@Test
void anthropicChatWithToolUse() {
server.reply("/v1/messages", 200, """
