This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch tui-ollama-tab in repository https://gitbox.apache.org/repos/asf/camel.git
commit 3f195298bffc23bf10fcd39c289c0b4ad5c0b290 Author: Claus Ibsen <[email protected]> AuthorDate: Thu Sep 17 12:24:12 2026 +0200 CAMEL-24794: camel-jbang - TUI Ollama tab counts cached prompt tokens once Ollama's prompt_eval_count is the whole prompt and prompt_eval_cached_count the part of it served from the KV cache, not an addition. The context fill added the two, reporting 77% for a prompt the runner's slot showed at 39%. The prefill rate now covers only the evaluated tokens (prompt minus cached), so a long prompt with a high cache hit no longer looks like a 24k tok/s prefill. Help text corrected accordingly; tui_get_ollama gains evaluatedTokens per request. Co-Authored-By: Claude Fable 5.1 <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../dsl/jbang/core/commands/tui/OllamaMonitor.java | 19 +++++++++++++++---- .../dsl/jbang/core/commands/tui/OllamaTab.java | 2 +- .../src/main/resources/tui/help/ollama.md | 14 ++++++++------ .../jbang/core/commands/tui/OllamaMonitorTest.java | 21 ++++++++++++++++++--- 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/OllamaMonitor.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/OllamaMonitor.java index ceaf4c63dd42..b7d0f485d183 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/OllamaMonitor.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/OllamaMonitor.java @@ -148,9 +148,17 @@ final class OllamaMonitor { int outputTokens, int cachedTokens, long prefillMs, long decodeMs, long loadMs, long totalMs, String doneReason, long contextSize) { - /** Everything the model had in front of it: prompt tokens evaluated plus those served from cache. */ + /** + * Everything the model had in front of it. Ollama's {@code prompt_eval_count} is the whole prompt; the cached + * count is the part of it served from the KV cache, not an addition. + */ long promptTokens() { - return (long) inputTokens + cachedTokens; + return inputTokens; + } + + /** Prompt tokens the runner actually had to evaluate this time. */ + long evaluatedTokens() { + return Math.max(0, (long) inputTokens - cachedTokens); } /** Share of the context window the prompt filled, or -1 when the window is unknown. */ @@ -161,8 +169,9 @@ final class OllamaMonitor { return (int) Math.min(100, promptTokens() * 100 / contextSize); } + /** Prefill speed over the tokens that were not served from cache. */ double prefillTokensPerSecond() { - return prefillMs > 0 ? inputTokens * 1000.0 / prefillMs : 0; + return prefillMs > 0 && evaluatedTokens() > 0 ? evaluatedTokens() * 1000.0 / prefillMs : 0; } double decodeTokensPerSecond() { @@ -202,7 +211,8 @@ final class OllamaMonitor { } double avgPrefillTokensPerSecond() { - return prefillMs > 0 ? inputTokens * 1000.0 / prefillMs : 0; + long evaluated = Math.max(0, inputTokens - cachedTokens); + return prefillMs > 0 && evaluated > 0 ? evaluated * 1000.0 / prefillMs : 0; } } @@ -986,6 +996,7 @@ final class OllamaMonitor { r.put("ttftMs", e.ttftMs()); r.put("coldStart", e.coldStart()); r.put("promptTokens", e.promptTokens()); + r.put("evaluatedTokens", e.evaluatedTokens()); r.put("contextSize", e.contextSize()); r.put("contextPercent", e.contextPercent()); if (e.doneReason() != null) { diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/OllamaTab.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/OllamaTab.java index e532e60b0e79..ec4b7521eacd 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/OllamaTab.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/OllamaTab.java @@ -345,7 +345,7 @@ class OllamaTab extends AbstractTab { RequestEntry last = s.lastRequest(); long ctx = model != null ? model.contextLength() : 0; if (last != null && ctx > 0) { - long used = (long) last.inputTokens() + last.outputTokens() + last.cachedTokens(); + long used = (long) last.inputTokens() + last.outputTokens(); int pct = (int) Math.min(100, used * 100 / ctx); lines.add(Line.from( Span.styled(" " + gaugeBar(pct, gaugeWidth) + " ", pct >= 90 ? Theme.error() : Theme.info()), diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/ollama.md b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/ollama.md index 63af33fc2a0f..da15e63a38dd 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/ollama.md +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/ollama.md @@ -22,9 +22,11 @@ one of them: size), Ollama starts a runner and loads the weights. Seconds to tens of seconds. A load of a second or more is shown as a **cold** start. 2. **Prefill** — the prompt (system prompt, tool definitions, history, - your question) is processed in one batch. Reported as *prefill - tokens per second*. Prompt tokens already in Ollama's cache from the - previous turn are skipped; they show as **cached**. + your question) is processed in one batch. Prompt tokens already in + Ollama's cache from the previous turn or tool step are skipped; they + show as **cached**. *Prefill tokens per second* counts only the + tokens that had to be evaluated, so a long prompt with a high cache + hit still has a short prefill. 3. **Decode** — the answer is generated one token at a time. Reported as *decode tokens per second*; this is the "typing speed" you see. @@ -110,11 +112,11 @@ One line per request, newest first: | TIME | When the request started | | SOURCE | `tui` for the AI panel, `route:<id>` for a Camel route | | MODEL | The model that answered | -| IN | Prompt tokens evaluated | +| IN | Prompt tokens, the whole prompt the model saw (CACHED is the part of it served from cache) | | OUT | Tokens generated | | CACHED | Prompt tokens served from Ollama's cache (`-` when none) | -| CTX | Share of the context window the prompt filled (IN plus CACHED against the window that served it) | -| PREFILL | Prompt tokens per second | +| CTX | Share of the context window the prompt filled (IN against the window that served it) | +| PREFILL | Evaluated prompt tokens per second (IN minus CACHED over the prefill time) | | DECODE | Generated tokens per second | | TTFT | Time to first token (load plus prefill), shown in yellow after a cold start | | TOTAL | Whole request as Ollama measured it | diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/OllamaMonitorTest.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/OllamaMonitorTest.java index bd8d7c1ecf3e..467422474974 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/OllamaMonitorTest.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/OllamaMonitorTest.java @@ -69,6 +69,19 @@ class OllamaMonitorTest { assertEquals("stop", e.doneReason()); } + @Test + void prefillRateCountsOnlyTokensNotServedFromCache() { + // a tool-loop step observed live: 12,774 prompt tokens of which 12,624 came from the cache, 532 ms prefill + OllamaMonitor monitor = new OllamaMonitor(); + monitor.recordRequest("m", new LlmClient.TokenUsage(12_774, 161, 12_935, 12_624, 532, 2_990, 0, 3_588), 0, + "stop"); + RequestEntry e = monitor.snapshot().lastRequest(); + assertEquals(150, e.evaluatedTokens()); + assertEquals(150 * 1000.0 / 532, e.prefillTokensPerSecond(), 0.1); + assertEquals(12_774, e.promptTokens()); + assertEquals(150 * 1000.0 / 532, monitor.snapshot().totals().avgPrefillTokensPerSecond(), 0.1); + } + @Test void coldStartIsDetectedFromLoadTime() { OllamaMonitor monitor = new OllamaMonitor(); @@ -150,15 +163,17 @@ class OllamaMonitorTest { monitor.recordRequest("m", new LlmClient.TokenUsage(4_000, 60, 4_060, 500, 900, 800, 0, 1800), 0, "stop"); RequestEntry first = monitor.snapshot().lastRequest(); assertEquals(32_768, first.contextSize()); - assertEquals(4_500, first.promptTokens()); - assertEquals(13, first.contextPercent()); + // prompt_eval_count is the whole prompt; the cached 500 are part of it, not on top + assertEquals(4_000, first.promptTokens()); + assertEquals(3_500, first.evaluatedTokens()); + assertEquals(12, first.contextPercent()); // the runner's slot wins over the model list when present monitor.updateSlot(slot(false, 0, System.currentTimeMillis())); monitor.recordRequest("m", new LlmClient.TokenUsage(20_000, 60, 20_060, 0, 900, 800, 0, 1800), 0, "stop"); assertEquals(262144, monitor.snapshot().lastRequest().contextSize()); assertEquals(7, monitor.snapshot().lastRequest().contextPercent()); - assertEquals(13, monitor.snapshot().totals().peakContextPercent()); + assertEquals(12, monitor.snapshot().totals().peakContextPercent()); assertEquals(0, monitor.snapshot().totals().compactions()); // a prompt a fifth smaller than the previous turn counts as a compaction
