This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
commit d889c9188c02a39c55984c22f12e432183c87fcf Author: Claus Ibsen <[email protected]> AuthorDate: Thu Oct 15 07:25:22 2026 +0200 ai-benchmark: 25 tool calls per step instead of 20 Of the 78 steps that hit 20 calls in r12-r14, 24 passed anyway and 30 still had an accepted write in their last 5 calls, so they were still making progress. The steps that pass need few calls (median 4, p95 13, p99 18), so a higher cap mostly gives the ones that were still writing room to finish. Runs before Oct 15 2026 had 20. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01STT6whBgK1AqsSsUKrnE8m --- ai-benchmark/agent_mcp_stepwise.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ai-benchmark/agent_mcp_stepwise.py b/ai-benchmark/agent_mcp_stepwise.py index 1fa104f..1548bf1 100755 --- a/ai-benchmark/agent_mcp_stepwise.py +++ b/ai-benchmark/agent_mcp_stepwise.py @@ -11,7 +11,7 @@ errors via camel_get_errors, diff size, reference checkpoint after each step. Env: BENCH_MODEL (ollama model), MCP_URL (default http://127.0.0.1:9090/mcp), BENCH_TAG (results folder name), BENCH_STEPS (the steps file, default steps.json), -BENCH_TOOL_CALLS (max tool calls per step, default 12). +BENCH_TOOL_CALLS (max tool calls per step, default 25; 20 until Oct 15 2026). """ import difflib, json, os, re, subprocess, sys, time, urllib.error, urllib.request sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @@ -23,7 +23,8 @@ MCP_URL = os.environ.get("MCP_URL", "http://127.0.0.1:9090/mcp";) HERE = os.path.dirname(os.path.abspath(__file__)) TAG = os.environ.get("BENCH_TAG", "mcp-" + MODEL.replace(":", "_").replace("/", "_")) OUT = os.path.join(HERE, "stepwise", TAG) -MAX_TOOL_CALLS = int(os.environ.get("BENCH_TOOL_CALLS", "20")) +# 25 since Oct 15 2026 (20 before, r12-r14): of the steps that hit 20, 38% were still writing in their last 5 calls +MAX_TOOL_CALLS = int(os.environ.get("BENCH_TOOL_CALLS", "25")) # the conversation grows over an example's steps (logs with stack traces are the bulk); at 32k a long example ran out # of context and the model stopped after a few words without a tool call (round 3, circuit-breaker steps 2-3) NUM_CTX = int(os.environ.get("BENCH_NUM_CTX", "65536"))
