This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
The following commit(s) were added to refs/heads/main by this push:
new 38755b2 ai-benchmark: retry a 500 from Ollama once, midnight in the
reload checks, score on the last reload that changed a route, cron + in nightly
s4
38755b2 is described below
commit 38755b20033e3cf902d9e23ad7d1039993671445
Author: Claus Ibsen <[email protected]>
AuthorDate: Wed Oct 28 22:29:37 2026 +0100
ai-benchmark: retry a 500 from Ollama once, midnight in the reload checks,
score on the last reload that changed a route, cron + in nightly s4
From the r23 analysis, 7 of the 34 ladder failures came from Ollama or the
harness:
- a 500 from Ollama ("qwen tool call parsing failed") ended the step; it is
now tried once more (filter s1, 3 steps)
- after_reload and the wait compared seconds of the day without the wrap
past midnight (stock-api s1, 1 step)
- a reload with changed:0 (a YAML repair) moved the start of the log window
past what the unchanged routes had
logged (file-processing s3, 1 step)
- the nightly s4 check refused 0+0+2+*+*+?, which camel-cron reads as 0 0 2
* * ? (3 steps)
Reference pass of the four examples: 14 of 14.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01STT6whBgK1AqsSsUKrnE8m
---
ai-benchmark/agent_mcp_stepwise.py | 32 +++++++++++++++++++++++++++-----
ai-benchmark/gen_stepwise.py | 3 ++-
2 files changed, 29 insertions(+), 6 deletions(-)
diff --git a/ai-benchmark/agent_mcp_stepwise.py
b/ai-benchmark/agent_mcp_stepwise.py
index 1f7dd0d..3c931f0 100755
--- a/ai-benchmark/agent_mcp_stepwise.py
+++ b/ai-benchmark/agent_mcp_stepwise.py
@@ -150,6 +150,13 @@ def tod(t):
return None
+def at_or_after(t, ref):
+ """t is at or after ref, as seconds of the day: a step that crosses
midnight has its reload at 23:59:50 and the
+ line it waits for at 00:00:00 (r23l-2 stock-api s1 failed on a right
route). Times more than 12 hours before ref
+ are taken as the next day, as in_step does."""
+ return t + (86400 if t < ref - 43200 else 0) >= ref
+
+
def is_reload(l):
m = (l.get("message") or l.get("msg") or "")
return "Routes reloaded summary" in m or "Error reloading routes" in m or
"Reloading properties" in m
@@ -318,7 +325,7 @@ def wait_for_check(mcp, name, chk, cfg, project, before,
reload_at):
if error_key(l) in seen_msgs:
return False
t = tod(l.get("time") or l.get("timestamp"))
- return reload_at is None or t is None or t >= reload_at
+ return reload_at is None or t is None or at_or_after(t, reload_at)
while time.time() - t0 < limit:
fresh = [msg(l) for l in log_lines(mcp, name, LOG_WINDOW) if
fresh_ok(l)]
logs_ok = all(sum(1 for m in fresh if re.search(rx, m, re.S)) >= need
for rx in regexes)
@@ -415,8 +422,13 @@ def score(step, project, cfg, mcp, name, before):
# the reload this step caused: what the step forbids is only
forbidden from here on, whatever the route
# logged while the model was still writing (the step before it may
have been the one provoking it)
# the last of them: a model that wrote twice in one step is judged
on the state it left behind
- times = [t for t in (tod(l.get("time") or l.get("timestamp")) for
l in fresh_reloads) if t is not None]
- reload_at = max(times) if times else None
+ # but a reload that changed no route ("changed:0", a repair the
YAML loader made anyway) leaves the routes
+ # as they were: what they logged before it still counts (r23l-5
file-processing s3: the files were moved
+ # on the first write, and the second write reloaded with
changed:0, so nothing could log after it)
+ changing = [l for l in fresh_reloads if "changed:0)" not in
(l.get("message") or l.get("msg") or "")]
+ times = [t for t in (tod(l.get("time") or l.get("timestamp")) for
l in (changing or fresh_reloads)) if t is not None]
+ # the last of them across midnight too: 23:59:58 is before 00:00:02
+ reload_at = max(times, key=lambda t: t + (86400 if t < max(times)
- 43200 else 0)) if times else None
break
time.sleep(gap)
# CAMEL-24886: probes are the README's curl: an HTTP request after the
reload, its status and body checked
@@ -452,7 +464,7 @@ def score(step, project, cfg, mcp, name, before):
if reload_at is None:
return True
t = tod(l.get("time") or l.get("timestamp"))
- return t is None or t >= reload_at
+ return t is None or at_or_after(t, reload_at)
fresh_recs = [l for l in recent if error_key(l) not in seen_msgs and
after_reload(l)]
fresh = [msg(l) for l in fresh_recs]
# round 2: log_regex may be a list (all must match), or absent (a step
with nothing to see in the log)
@@ -640,7 +652,17 @@ def main():
answer = "(reference)"
while not REFERENCE:
try:
- data, secs = ollama_chat(messages, tools)
+ try:
+ data, secs = ollama_chat(messages, tools)
+ except urllib.error.HTTPError as e:
+ # a 500 from Ollama is its own parse failure of the
answer ("qwen tool call parsing failed:
+ # expected element type <function> but have
<parameter>"), not the model's step: r23 lost three
+ # filter s1 steps on the first turn, on a prompt that
passed 7 of 7 in r22. One more try.
+ if e.code != 500:
+ raise
+ trace.write(json.dumps({"error": str(e), "retry":
True}) + "\n"); trace.flush()
+ time.sleep(5)
+ data, secs = ollama_chat(messages, tools)
except Exception as e:
trace.write(json.dumps({"error": str(e)}) + "\n"); break
msg = data["message"]; tokens += data.get("eval_count", 0)
diff --git a/ai-benchmark/gen_stepwise.py b/ai-benchmark/gen_stepwise.py
index 1f37f80..ce1a8ec 100644
--- a/ai-benchmark/gen_stepwise.py
+++ b/ai-benchmark/gen_stepwise.py
@@ -120,7 +120,8 @@ EXAMPLES.append({
"check": {"props_regex": "report\\.schedule\\s*=", "file_regex":
"\\{\\{report\\.schedule\\}\\}", "log_regex": "120 T-shirts and 45 mugs"},
"reference": {NR: route("nightly-report", CRON_P, log(MSG3)),
"application.properties": "report.schedule=0/10 * * * *
?\ninventory.tshirt=120\ninventory.mug=45\n"}},
{"request": "Change the schedule to 0 0 2 * * ? so the report runs at
two in the morning every night.",
- "check": {"props_regex": "report\\.schedule\\s*=\\s*0 0 2 \\* \\*
\\?"},
+ # camel-cron reads + as a space (CronComponent), so 0+0+2+*+*+? is
the same schedule (r23: 3 of 3 fails)
+ "check": {"props_regex": "report\\.schedule\\s*=\\s*0[ +]0[ +]2[
+]\\*[ +]\\*[ +]\\?"},
"reference": {"application.properties": "report.schedule=0 0 2 * *
?\ninventory.tshirt=120\ninventory.mug=45\n"}},
]})
