This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
The following commit(s) were added to refs/heads/main by this push:
new 6fcee41 ai-benchmark: a check that cannot be read fails its step, the
model is told when a passed step's files are replaced, and jms step 2 checks
that the order stays the body
6fcee41 is described below
commit 6fcee4182ec72072b9566a0697b7b046b06c981b
Author: Claus Ibsen <[email protected]>
AuthorDate: Thu Oct 22 08:47:01 2026 +0200
ai-benchmark: a check that cannot be read fails its step, the model is told
when a passed step's files are replaced, and jms step 2 checks that the order
stays the body
From the r21 analysis (Oct 22 2026):
- read() survives a directory where a file was expected, globbed checks
read files only, and a scoring error
fails the step instead of ending the example (r21n-7 ai-ticket-classifier
stopped after step 1)
- the checkpoint of a failed step empties the example's output folders
(clear_dirs: tickets-out)
- after a passed step whose files the checkpoint replaces with the
reference, the model is told to read a file
again (it went on from memory with files that had changed)
- jms step 2 checks variableReceive in reserve-order: the answer in a
variable, so the order stays the body
(r21n-1 passed with the reply replacing the body)
- summarize_stepwise.py warns when a run recorded fewer steps than the
example has
Reference passes with the change: new set 19/19, ladder 71/71.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01STT6whBgK1AqsSsUKrnE8m
---
ai-benchmark/agent_mcp_stepwise.py | 33 +++++++++++++++++++++++++++++----
ai-benchmark/gen_stepwise.py | 10 ++++++++--
ai-benchmark/summarize_stepwise.py | 10 ++++++++++
3 files changed, 47 insertions(+), 6 deletions(-)
diff --git a/ai-benchmark/agent_mcp_stepwise.py
b/ai-benchmark/agent_mcp_stepwise.py
index 9c6af35..3d1ef6e 100755
--- a/ai-benchmark/agent_mcp_stepwise.py
+++ b/ai-benchmark/agent_mcp_stepwise.py
@@ -102,7 +102,8 @@ def to_ollama_tools(mcp_tools, names):
def read(path):
try:
return open(path).read()
- except FileNotFoundError:
+ except OSError:
+ # also a directory where a file was expected (r21n-7: the model's toD
made tickets-out/shipping/T-1.json one)
return ""
@@ -273,7 +274,7 @@ def file_matches(project, fname, rx):
category folder of ticket-classifier, which the model's answer picks, Oct
18); else project_file."""
if "*" in fname:
import glob
- return any(re.search(rx, read(f) or "") for f in
glob.glob(os.path.join(project, fname)))
+ return any(re.search(rx, read(f) or "") for f in
glob.glob(os.path.join(project, fname)) if os.path.isfile(f))
return bool(re.search(rx, project_file(project, fname) or ""))
@@ -691,7 +692,15 @@ def main():
if isinstance(data, dict) and data.get("name"):
name = data["name"]
time.sleep(8)
- res, route_after, props_after = score(step, project, cfg, mcp,
name, before)
+ try:
+ res, route_after, props_after = score(step, project, cfg, mcp,
name, before)
+ except Exception as e:
+ # a check that cannot be read fails the step instead of ending
the example (r21n-7 ended after step 1)
+ print(f"step{sid}: scoring failed: {e!r}", file=log,
flush=True)
+ res = {"file_ok": False, "props_ok": False, "log_ok": False,
"errors": 0, "changed_lines": 0, "ok": False,
+ "ok_final": False, "ok_lastwrite": False,
"score_error": repr(e)}
+ route_after = read(os.path.join(project, cfg["route_file"]))
+ props_after = read(os.path.join(project, cfg.get("props_file",
"application.properties")))
res.update({"step": sid, "request": step["request"], "tool_calls":
calls, "writes": writes, "refused_writes": refused,
"seconds": round(time.time() - t0, 1), "tokens":
tokens, "answer": answer[:400]})
results.append(res)
@@ -722,9 +731,19 @@ def main():
restored = not res["ok"] and restore_seeds(project, cfg)
if restored:
print(f"step{sid}: checkpoint restored seed files",
file=log, flush=True)
+ if not res["ok"]:
+ # what a failed step wrote into an output folder of the
example goes too: r21n-7 left a directory
+ # named tickets-out/shipping/T-1.json, and every later
write of that file failed
+ for d in cfg.get("clear_dirs", []):
+ out = os.path.join(project, d)
+ if os.path.isdir(out) and os.listdir(out):
+ shutil.rmtree(out, ignore_errors=True)
+ os.makedirs(out, exist_ok=True)
+ print(f"step{sid}: checkpoint emptied {d}",
file=log, flush=True)
+ differs = removed or restored or
any(read(os.path.join(project, f)) != c for f, c in step["reference"].items())
if FAST:
# files that already equal the reference are not written
again: no reload, nothing to wait for
- if removed or restored or any(read(os.path.join(project,
f)) != c for f, c in step["reference"].items()):
+ if differs:
seen = error_snapshot(mcp, name)[2]
apply_reference(project, cfg, step["reference"], mcp,
name)
if restart_after:
@@ -750,6 +769,12 @@ def main():
time.sleep(6)
if not res["ok"]:
messages.append({"role": "user", "content": "I fixed that
step myself; the files now contain the correct version. Continue with the next
request."})
+ elif differs:
+ # the model's own version passed, and the checkpoint put
the reference in its place: without a word
+ # the model went on from memory with files that had
changed (r21: run-nightly-report step 3 sent
+ # edits for its own version of the route, jms step 3
changed a Java class the restored route no
+ # longer used)
+ messages.append({"role": "user", "content": "Your version
of that step passed. I replaced the files with my own version of that step, so
read a file again before you change it. Continue with the next request."})
json.dump(results, open(os.path.join(OUT, "results.json"), "w"),
indent=1)
finally:
data, raw = jcall(mcp, "camel_control", {"name": name, "action":
"stop"})
diff --git a/ai-benchmark/gen_stepwise.py b/ai-benchmark/gen_stepwise.py
index 6c745c0..d9d2ca9 100644
--- a/ai-benchmark/gen_stepwise.py
+++ b/ai-benchmark/gen_stepwise.py
@@ -889,7 +889,10 @@ EXAMPLES.append({
"check": {"log_regex": ["Order ORD-1001 put on the queue", "Reserving
ORD-1001 for customer C-482"]},
"reference": {JR: jr_s2, "application.properties": JR_CF}, "restart":
True},
{"request": "Add a route stock-service on the queue stock that answers
whether an order (the JSON body) can be reserved: \"reserved\" when stock.json
has enough of every line, otherwise \"not enough <sku>\" for the first line
that is short. In reserve-order, ask the stock service with request-reply, keep
the order as the body, and log \"Stock service answered for ${header.orderId}:
<the answer>\".",
- "check": {"log_regex": ["Stock service answered for ORD-1001:
reserved", "Stock service answered for ORD-1003: not enough CAMEL-CAP"]},
+ # the order stays the body: the answer goes to a variable
(variableReceive, as the jms docs show), where a
+ # reply that replaced the body passed before (r21n-1)
+ "check": {"log_regex": ["Stock service answered for ORD-1001:
reserved", "Stock service answered for ORD-1003: not enough CAMEL-CAP"],
+ "file_regex": "id: reserve-order(?:(?!-
route:)[\\s\\S])*variableReceive"},
"reference": {JR: jr_s3}},
{"request": "An order the stock service cannot reserve must fail:
redeliver it 3 times, one second apart, and then move the original order to the
queue orders.dead, where a route dead-orders logs \"Dead order
${header.orderId}\".",
"check": {"file_regex": "maximumRedeliveries", "log_regex": "Dead
order ORD-1003", "errors_ok": True},
@@ -1084,7 +1087,7 @@ tc_s4 = TC_BEANS + route("classify-tickets",
FILE_TICKETS, TC_RECEIVED + TC_BREA
TC_CATEGORY = "(?i)Ticket %s is (shipping|billing|account)\\b"
EXAMPLES.append({
"name": "ai-ticket-classifier", "route_file": TC, "props_file":
"application.properties", "wait_seconds": 20,
- "seed": "ai-ticket-classifier", "steps_dir": NEW,
+ "seed": "ai-ticket-classifier", "steps_dir": NEW, "clear_dirs":
["tickets-out"],
"initial": {TC: tc_s1, "application.properties": TC_PROPS},
"steps": [
{"request": "Ask the local model for the category of each ticket:
shipping, billing or account. Use the langchain4j-chat component with an Ollama
chat model (dev.langchain4j.model.ollama.OllamaChatModel) for the model and
base URL in application.properties. Keep the ticket as the body, the category
in a variable category, and log \"Ticket ${body[ticketId]} is
${variable.category}\".",
@@ -1132,6 +1135,9 @@ def main():
if ex.get("infra"):
# services the example needs, started with `camel infra run`
before the app (the sql example's postgres)
cfg["infra"] = ex["infra"]
+ if ex.get("clear_dirs"):
+ # output folders the checkpoint of a failed step empties
+ cfg["clear_dirs"] = ex["clear_dirs"]
if ex.get("peer"):
cfg["peer"] = {"project": ex["peer"]["project"], "name":
ex["peer"]["name"]}
pd = os.path.join(HERE, ex["peer"]["project"])
diff --git a/ai-benchmark/summarize_stepwise.py
b/ai-benchmark/summarize_stepwise.py
index 7ddde33..171805a 100644
--- a/ai-benchmark/summarize_stepwise.py
+++ b/ai-benchmark/summarize_stepwise.py
@@ -13,6 +13,7 @@ names = sorted(os.path.splitext(f)[0] for f in
os.listdir(steps_dir) if f.endswi
print(f"| example | steps | passes per step ({' '.join(tags)}) | all steps
passed | last version right | final state right |")
print("|---|---|---|---|---|---|")
total = 0; possible = 0; clean = 0; final = 0; lastw = 0
+missing = []
for n in names:
per = []; runs = []
for t in tags:
@@ -23,6 +24,13 @@ for n in names:
if not runs:
continue
steps = max(len(r) for r in runs)
+ # a run that recorded fewer steps than the example has: said, so a short
run is not read as a short example
+ # (r21n-7 recorded 1 of the 3 steps of ai-ticket-classifier, and the round
read "9 of 17")
+ defined = len(json.load(open(os.path.join(steps_dir, n +
".json"))).get("steps", []))
+ for t, r in zip([t for t in tags if
os.path.exists(os.path.join("stepwise", t, n, "results.json"))], runs):
+ if len(r) < defined:
+ missing.append(f"{n} {t}: {len(r)} of {defined} steps recorded")
+ steps = max(steps, defined)
cols = []
for s in range(steps):
ok = sum(1 for r in runs if s < len(r) and r[s]["ok"]);
cols.append(f"{ok}/{len(runs)}"); total += ok; possible += len(runs)
@@ -33,3 +41,5 @@ for n in names:
print(f"| {n} | {steps} | {' '.join(cols)} | {allok}/{len(runs)} |
{lw}/{n_steps} | {fin}/{n_steps} |")
print(f"\nsteps passed: {total} of {possible}; last version right: {lastw} of
{possible}; final state right: {final} of"
f" {possible}; runs with every step passed: {clean}")
+for m in missing:
+ print(f"WARNING: {m}")
