This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
commit 4fae7f9590ec10dd6e340f1af003135663b31245 Author: Claus Ibsen <[email protected]> AuthorDate: Wed Oct 14 21:49:49 2026 +0200 ai-benchmark: BENCH_FAST counts only the errors after a step began, BENCH_SERVER_INSTRUCTIONS, and data dirs for the ladder - With BENCH_FAST, an error logged before the step began, or an error record with an earlier timestamp, belongs to the step before, however late it came in. The shorter quiet after a reference write let one land in the next step's count (r13, one case in three runs). - BENCH_SERVER_INSTRUCTIONS=1 appends the instructions from the MCP server's initialize answer (CAMEL-25646) to the system prompt, as agent hosts show them to the model. - connect-file-processing, transform-csv-to-json and fail-well-error-handling declare the folders their routes write into as camel.jbang.dataDir (CAMEL-25670). With --source-dir, dev mode reloaded connect-file-processing every 2 s for its own files. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01STT6whBgK1AqsSsUKrnE8m --- ai-benchmark/agent_mcp_stepwise.py | 33 ++++++++++++++++++++++++++++++--- ai-benchmark/gen_stepwise.py | 9 ++++++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/ai-benchmark/agent_mcp_stepwise.py b/ai-benchmark/agent_mcp_stepwise.py index 0c40c46..1fa104f 100755 --- a/ai-benchmark/agent_mcp_stepwise.py +++ b/ai-benchmark/agent_mcp_stepwise.py @@ -161,6 +161,9 @@ FAST_POLL = float(os.environ.get("BENCH_FAST_POLL", "0.5")) # seconds betwee FAST_GRACE = float(os.environ.get("BENCH_FAST_GRACE", "2")) # after the checks match: time for errors that follow FAST_QUIET = float(os.environ.get("BENCH_FAST_QUIET", "1.5")) # after a reference write: the log quiet this long FAST_NEG_FULL = os.environ.get("BENCH_FAST_NEG_FULL", "1") == "1" # a step that forbids output waits the full time +# BENCH_SERVER_INSTRUCTIONS=1: append the instructions the MCP server sends in its initialize answer (CAMEL-25646) to +# the system prompt, as agent hosts such as Claude Code show them to the model +SERVER_INSTRUCTIONS = os.environ.get("BENCH_SERVER_INSTRUCTIONS") == "1" def infra_start(services): @@ -422,11 +425,29 @@ def score(step, project, cfg, mcp, name, before): result["log_ok"] = False # round 2: errors are counted per step (the log window and the error file keep an earlier step's stack traces) seen = before.get("error_records") or set() - result["errors"] = sum(1 for l in lines if isinstance(l, dict) and lvl(l) == "ERROR" and error_key(l) not in seen) + began = before.get("began") + + def in_step(l): + # BENCH_FAST: an error logged before the step began is the step before's, however late it came in + if not FAST or began is None: + return True + t = tod(l.get("time") or l.get("timestamp")) + b = tod(time.strftime("%H:%M:%S", time.localtime(began))) + if t is None or b is None: + return True + return t + (86400 if t < b - 43200 else 0) >= b - 0.5 + result["errors"] = sum(1 for l in lines if isinstance(l, dict) and lvl(l) == "ERROR" and error_key(l) not in seen + and in_step(l)) data, _ = jcall(mcp, "camel_get_errors", {"name": name}) if isinstance(data, dict): errs = data.get("errors", []) or [] - result["errors"] += max(0, len(errs) - before.get("error_count", 0)) + if FAST and began is not None: + # the error records carry their time (epoch millis): only the ones at or after the step began + new = errs[before.get("error_count", 0):] + result["errors"] += sum(1 for e in new if not isinstance(e, dict) or not isinstance(e.get("timestamp"), (int, float)) + or e["timestamp"] >= (began - 0.5) * 1000) + else: + result["errors"] += max(0, len(errs) - before.get("error_count", 0)) # what they were, not just how many: a step that fails on errors alone cannot be judged from a count result["error_detail"] = [str(e)[:300] for e in errs[before.get("error_count", 0):]][:5] diff = list(difflib.unified_diff(before["route"].splitlines(), route.splitlines(), lineterm="", n=0)) @@ -465,7 +486,8 @@ def main(): project = os.path.join(HERE, project) os.makedirs(OUT, exist_ok=True) log = open(os.path.join(OUT, "run.log"), "a") - mcp = McpClient(MCP_URL); mcp.initialize(); all_tools = mcp.list_tools() + mcp = McpClient(MCP_URL); init = mcp.initialize(); all_tools = mcp.list_tools() + instructions = ((init or {}).get("result") or {}).get("instructions") if isinstance(init, dict) else None tools = to_ollama_tools(all_tools, BARE_TOOLS if BARE else SHARED + EXTRA) print(f"tools: {[t['function']['name'] for t in tools]}", file=log, flush=True) @@ -517,6 +539,10 @@ def main(): time.sleep(6) base_system = SYSTEM.replace("{directory}", project).replace("{name}", name) + if SERVER_INSTRUCTIONS: + if instructions: + base_system += "\nThe Camel MCP server's own instructions:\n" + instructions.strip() + "\n" + print(f"server instructions: {len(instructions or '')} chars", file=log, flush=True) if BARE: # the guidelines about tools it does not have would only confuse it base_system = "\n".join(l for l in base_system.splitlines() @@ -541,6 +567,7 @@ def main(): before = {"route": read(os.path.join(project, cfg["route_file"])), "props": read(os.path.join(project, cfg.get("props_file", "application.properties")))} before["error_records"], before["error_count"], before["reload_records"], before["log_records"] \ = error_snapshot(mcp, name) + before["began"] = time.time() trace = open(os.path.join(OUT, f"step{sid}.trace.jsonl"), "w") messages.append({"role": "user", "content": step["request"]}) calls = 0; tokens = 0; t0 = time.time(); writes = 0; refused = 0; answer = "" diff --git a/ai-benchmark/gen_stepwise.py b/ai-benchmark/gen_stepwise.py index 66d97ce..4b5eb65 100644 --- a/ai-benchmark/gen_stepwise.py +++ b/ai-benchmark/gen_stepwise.py @@ -14,6 +14,9 @@ import datetime, json, os, shutil, sys HERE = os.path.dirname(os.path.abspath(__file__)) STEPS_DIR = os.path.join(HERE, "steps-ladder") +# CAMEL-25670: dev mode with --source-dir reloads the routes for every file they write into the project's own folders +# (connect-file-processing reloaded every 2 s, r13); camel.jbang.dataDir leaves those folders out (Camel 4.24) +DATA_DIR = "camel.jbang.dataDir=" PROJECTS = os.path.join(HERE, "stepwise-ladder") SEEDS = os.path.join(HERE, "seed") @@ -396,7 +399,7 @@ csv_s5 = route("csv-to-json", FILE_INBOX_CSV, UNMARSHAL_CSV + split_body(CSV_HDR EXAMPLES.append({ "name": "transform-csv-to-json", "route_file": CSV, "props_file": "application.properties", "wait_seconds": 8, "seed": "transform-csv-to-json", - "initial": {CSV: csv_s1, "application.properties": ""}, + "initial": {CSV: csv_s1, "application.properties": DATA_DIR + "inbox,outbox\n"}, "steps": [ {"request": "Unmarshal the CSV with the csv data format so the body becomes a list of maps keyed by the header row (useMaps and captureHeaderRecord), and log \"Invoices as a list of maps: ${body}\".", "check": {"file_regex": "csv", "log_regex": "Invoices as a list of maps: \\[\\{invoiceId=INV-2001"}, @@ -542,7 +545,7 @@ eh_s4 = DLC + ON_DECLINED + CHECKOUT + "\n" + provider(FLAKY + DECLINED, True) + EXAMPLES.append({ "name": "fail-well-error-handling", "route_file": EH, "props_file": "application.properties", "wait_seconds": 12, "seed": "fail-well-error-handling", - "initial": {EH: eh_s1, "application.properties": ""}, + "initial": {EH: eh_s1, "application.properties": DATA_DIR + "orders,parked\n"}, "steps": [ {"request": "Add a top-level errorHandler with a deadLetterChannel to direct:parked (maximumRedeliveries 2, redeliveryDelay 1000, retryAttemptedLogLevel WARN) and a route parked from direct:parked that logs \"Order ${body[orderId]} parked for manual review: ${exception.message}\", so the declined order is parked after the retries instead of dropped with a stack trace.", "check": {"file_regex": "deadLetterChannel", "log_regex": ["Order ORD-1003 parked for manual review: card declined", "Payment for ORD-1001 charged, 2 line\\(s\\)"], "errors_ok": True}, @@ -578,7 +581,7 @@ fp_s5 = ON_INVALID + COURIER + "\n" + route("invoices", INBOX_JSON_FAILED, UNMAR EXAMPLES.append({ "name": "connect-file-processing", "route_file": FP, "props_file": "application.properties", "wait_seconds": 8, "seed": "connect-file-processing", - "initial": {FP: fp_s1, "application.properties": ""}, + "initial": {FP: fp_s1, "application.properties": DATA_DIR + "samples,inbox,archive\n"}, "steps": [ {"request": "The invoices route must only take the JSON files: include .*\\.json, and move each processed file to the done directory instead of the default .camel directory. The driver's note note-2005.txt must stay untouched in inbox.", "check": {"file_regex": "move: done", "file_regex2": "include", "log_regex": "Received invoice-2004.json", "log_not_regex": "Received note-2005", "files": {"inbox/note-2005.txt": "Handwritten note", "done/invoice-2001.json": "INV-2001"}},
