This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
commit 826d78e0000dda5a97e635f9db1b6e1c6fde6d1a Author: Claus Ibsen <[email protected]> AuthorDate: Fri Nov 6 09:43:52 2026 +0100 ai-benchmark: test-citrus-orders step 3 accepts the empty order in a file under test/ that the test reads r28n-5, -6 and -7 wrote the empty order to test/empty-lines-order.json, and the test sent it with resource: file: "../test/empty-lines-order.json", as the test of step 1 sends sample-order.json. The test passed, both probes passed (400 with the error, 201 for a good order) and there were no errors. But the check wanted "lines": [] in the test file itself, so the step failed. A new check, test_data {test: regex}, takes the regex in the test, or in a file under the test's folder that the test names by a path from that folder. Step 3 uses it for "lines": []. The error text stays a check on the test file. The start state, a test that sends no empty order, a file the test does not name, and a file outside test/ still fail. A re-score of the step 3 runs, with the test and its files rebuilt from the traces: r28n goes from 3/7 to 6/7 (r28n-2 failed on the route and stays failed), r26n stays 6/7. The reference pass passes all 3 steps, the negative pass fails all 3. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_016Y4dDi5iQNgtHrDdz2NpwF --- ai-benchmark/agent_mcp_stepwise.py | 24 ++++++++++++++++++++++++ ai-benchmark/gen_stepwise.py | 5 ++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/ai-benchmark/agent_mcp_stepwise.py b/ai-benchmark/agent_mcp_stepwise.py index 432a99e..02ee64b 100755 --- a/ai-benchmark/agent_mcp_stepwise.py +++ b/ai-benchmark/agent_mcp_stepwise.py @@ -353,6 +353,25 @@ def file_matches(project, fname, rx): return bool(re.search(rx, project_file(project, fname) or "")) +def test_reads(project, test, rx): + """test_data: whether a test holds the regex, or reads a file under its folder that holds it. The test names that + file by a path from its own folder, as it names "../sample-order.json" of the project (r28n-5, -6 and -7 + test-citrus-orders s3 put the empty order in test/empty-lines-order.json and sent it with resource: file: + "../test/empty-lines-order.json"; the test passed).""" + path = os.path.join(project, test) + text = read(path) or "" + if re.search(rx, text): + return True + home = os.path.dirname(path) + named = {os.path.normpath(os.path.join(home, w)) for w in re.findall(r"[^\s\"'(),]+", text)} + for root, dirs, files in os.walk(home): + for f in files: + p = os.path.normpath(os.path.join(root, f)) + if p != os.path.normpath(path) and p in named and re.search(rx, read(p) or ""): + return True + return False + + def server_home(cfg): """The folder of the example's seeded infra service (the files on its SFTP server), or None.""" for svc in cfg.get("infra_seed") or {}: @@ -688,6 +707,11 @@ def score(step, project, cfg, mcp, name, before, log=None): # a name with * is a glob if not file_matches(project, fname, rx): result["file_ok"] = False + # test_data {test: regex}: what a test sends holds the regex, in the test itself or in a file under its folder + # that it reads (a request body in resource: file:) + for fname, rx in (chk.get("test_data") or {}).items(): + if not test_reads(project, fname, rx): + result["file_ok"] = False # the files on the server of a seeded infra service: server_files {path or glob: regex} must be there and match, # server_not_files [path or glob] must be gone (a batch the route moved away) home = server_home(cfg) diff --git a/ai-benchmark/gen_stepwise.py b/ai-benchmark/gen_stepwise.py index e3a3309..74b6107 100644 --- a/ai-benchmark/gen_stepwise.py +++ b/ai-benchmark/gen_stepwise.py @@ -1325,8 +1325,11 @@ EXAMPLES.append({ "check": {"test_run": CO_TEST, "files": {CO_TEST: "(?s)(?=.*\\b400\\b)(?=.*an order needs a customer)"}}, "reference": {CO_TEST: CO_T2}}, {"request": "An order with no lines is accepted today (201), which is a bug. Work test first: add a case to the test where an order whose lines are empty gets the answer 400 with {\"error\": \"an order needs at least one line\"}, then fix the route so that the test passes.", + # the empty order may be in the test or in a file under test/ that the test reads, as the test of step 1 reads + # sample-order.json (r28n-5, -6 and -7 wrote test/empty-lines-order.json; the test passed) "check": {"test_run": CO_TEST, "file_regex": "at least one line", - "files": {CO_TEST: "(?s)(?=.*\"lines\"\\s*:\\s*\\[\\s*\\])(?=.*an order needs at least one line)"}, + "files": {CO_TEST: "an order needs at least one line"}, + "test_data": {CO_TEST: "\"lines\"\\s*:\\s*\\[\\s*\\]"}, "probes": [POST("/orders", CO_EMPTY, 400, "an order needs at least one line"), POST("/orders", CO_GOOD, 201, "accepted")]}, "reference": {CO: co_state(True), CO_TEST: CO_T3}}, ]})
