This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
The following commit(s) were added to refs/heads/main by this push:
new 7a0f6c4 ai-benchmark: enrich step 2 checks that the order stays the
body, the REST 200 probe wants one order, and the traces keep more of a tool
answer
7a0f6c4 is described below
commit 7a0f6c452913ccff974a1ac32ba59d86e1f6703a
Author: Claus Ibsen <[email protected]>
AuthorDate: Wed Oct 21 07:54:31 2026 +0200
ai-benchmark: enrich step 2 checks that the order stays the body, the REST
200 probe wants one order, and the traces keep more of a tool answer
- route-enrich-and-dedupe step 2: file_regex2 "Order ${body[orderId]} for",
as the request says to keep the order as
the body; in r20 the 8 runs whose log was right had replaced the body
with the customer.
- contracts-rest-code-first step 1: the 200 probe's body regex starts at a
JSON object, so an answer of the whole list
([{...}]) no longer passes.
- The traces keep 4000 characters of each tool answer (was 800), so the
hints in the answers can be seen in the analysis.
Reference pass of the new set: 19 of 19.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01STT6whBgK1AqsSsUKrnE8m
---
ai-benchmark/agent_mcp_stepwise.py | 6 +++++-
ai-benchmark/gen_stepwise.py | 8 ++++++--
2 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/ai-benchmark/agent_mcp_stepwise.py
b/ai-benchmark/agent_mcp_stepwise.py
index 4beeda6..9c6af35 100755
--- a/ai-benchmark/agent_mcp_stepwise.py
+++ b/ai-benchmark/agent_mcp_stepwise.py
@@ -368,6 +368,10 @@ def probe(pr):
return out
+# how much of each tool answer the trace keeps: 800 cut the hints the analysis
needs to see (r20)
+TRACE_RESULT_CHARS = 4000
+
+
def body_matches(rx, body):
"""The check is about what the route answers, not how it spaces it: a
pretty printed JSON body is the same
answer as a compact one, so the regex is tried against the text and
against the body re-serialized compactly."""
@@ -673,7 +677,7 @@ def main():
before["last_write"] = error_snapshot(mcp, name)
chars = len(out)
out = out[:CATALOG_RESULT_CAP if fn["name"] ==
"camel_catalog_doc" else TOOL_RESULT_CAP]
- trace.write(json.dumps({"tool": fn["name"], "ms": ms,
"args": {k: (v if k != "content" else v[:1500]) for k, v in args.items()},
"result": out[:800], "chars": chars, "shown": len(out)}) + "\n"); trace.flush()
+ trace.write(json.dumps({"tool": fn["name"], "ms": ms,
"args": {k: (v if k != "content" else v[:1500]) for k, v in args.items()},
"result": out[:TRACE_RESULT_CHARS], "chars": chars, "shown": len(out)}) +
"\n"); trace.flush()
messages.append({"role": "tool", "content": out,
"tool_name": fn["name"]})
continue
answer = msg.get("content") or ""
diff --git a/ai-benchmark/gen_stepwise.py b/ai-benchmark/gen_stepwise.py
index 6ac4ebe..6c745c0 100644
--- a/ai-benchmark/gen_stepwise.py
+++ b/ai-benchmark/gen_stepwise.py
@@ -947,7 +947,10 @@ EXAMPLES.append({
"check": {"file_regex": "idempotentConsumer", "log_regex": ["Order
ORD-1001 accepted", "Order ORD-1003 accepted", "Order ORD-1001 is a duplicate,
skipped"]},
"reference": {ED: ed_s2}},
{"request": "Enrich each accepted order with its customer: read
customers/<customer>.json with pollEnrich, keep the order as the body, and log
\"Order ${body[orderId]} for <name> (<tier>)\" with the customer's name and
tier.",
- "check": {"file_regex": "pollEnrich", "log_regex": ["Order ORD-1001
for Anna Jensen \\(gold\\)", "Order ORD-1003 for Emily Carter \\(bronze\\)"]},
+ # the order stays the body, as the request says: the log reads its id
with ${body[orderId]} (r20: the 8 runs
+ # whose log was right had replaced the body with the customer)
+ "check": {"file_regex": "pollEnrich", "file_regex2": "Order
\\$\\{body\\[orderId\\]\\} for",
+ "log_regex": ["Order ORD-1001 for Anna Jensen \\(gold\\)",
"Order ORD-1003 for Emily Carter \\(bronze\\)"]},
"reference": {ED: ed_s3}},
{"request": "Copy each accepted order as JSON to audit/<orderId>.json
with a wire tap, so the copy does not hold the order up, and log \"Audited
<orderId>.json\" when a copy is written.",
"check": {"file_regex": "wireTap", "log_regex": "Audited
ORD-1001.json", "files": {"audit/ORD-1001.json": "ORD-1001"}},
@@ -1031,7 +1034,8 @@ EXAMPLES.append({
"initial": {RC: rc_state(False, False, False), "application.properties":
PORT_PROPS},
"steps": [
{"request": "Add GET /orders/{id} to the REST API: answer the order
with that id from orders.json, or 404 with {\"error\": \"unknown order
<id>\"}.",
- "check": {"file_regex": "\\{id\\}", "probes":
[GET("/orders/ORD-1002", 200, "\"customer\":\"C-207\""),
GET("/orders/ORD-9999", 404, "unknown order ORD-9999")]},
+ # the one order, not the list that holds it (r20: an answer of
[{...}] matched)
+ "check": {"file_regex": "\\{id\\}", "probes":
[GET("/orders/ORD-1002", 200, "^\\s*\\{.*\"customer\":\"C-207\""),
GET("/orders/ORD-9999", 404, "unknown order ORD-9999")]},
"reference": {RC: rc_state(True, False, False)}},
{"request": "Add POST /orders that binds the JSON of a new order with
the json binding to a Java record camel.example.Order (orderId, customer,
country, status and lines; each line has sku, qty and price). Answer 400 with
{\"error\": \"an order needs at least one line\"} when the order has no lines;
otherwise log \"Order ${body.orderId} created with ${body.lines.size()}
line(s)\" and answer 201 with the order. A new Java class needs a restart of
the integration to be compiled; it [...]
"check": {"files": {"Order.java": "record Order"}, "probes":
[POST("/orders", RC_NEW, 201, "\"orderId\":\"ORD-1004\""), POST("/orders",
RC_EMPTY, 400, "at least one line")],
