This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
commit f3bb73934ed686a0b58d0d57ec80789d36a57bcd Author: Claus Ibsen <[email protected]> AuthorDate: Fri Nov 6 09:55:06 2026 +0100 ai-benchmark: results.json keeps the times of every counted error record, and when the last version began error_detail keeps 300 characters of at most 5 records. The endpoint URIs come before elapsed in a record, so 323 of the 353 records of r28 lost it, and the re-score of r28 had to take the time a record failed as the latest time its exchange began. It also had to guess when the last write was sent, from the file times of the trace. results.json now has, for each step: - error_times: uid, timestamp and elapsed of every error record the step counts; - errors_after_last_write_times and errors_older_version_times: the same for the records counted after the last write, and for those left out as the version before; - last_write_sent, last_write_at and last_write_spare: when the last write was sent, when the last version began (the reload record, or the time the write was sent), and the seconds to spare that began_before used. So a re-score can recompute the errors after the last write exactly. The ERROR lines of the log that a count includes are the count less the records. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_016Y4dDi5iQNgtHrDdz2NpwF --- ai-benchmark/agent_mcp_stepwise.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/ai-benchmark/agent_mcp_stepwise.py b/ai-benchmark/agent_mcp_stepwise.py index 79e2f00..8366d01 100755 --- a/ai-benchmark/agent_mcp_stepwise.py +++ b/ai-benchmark/agent_mcp_stepwise.py @@ -212,6 +212,11 @@ def reload_logged(reloads, t0, t1): return max(found) if found else None +def record_times(e): + """The uid, timestamp (epoch millis, when it failed) and elapsed (millis its exchange had run) of an error record.""" + return {k: e.get(k) for k in ("uid", "timestamp", "elapsed")} if isinstance(e, dict) else {} + + def records_since(errs, t, count): """The records of the error file that came after a snapshot of count records taken at t (epoch seconds): the ones that failed at or after t, with half a second to spare. The file lists the newest first and keeps only the newest 3 @@ -814,6 +819,9 @@ def score(step, project, cfg, mcp, name, before, log=None): result["errors"] += len(new) # what they were, not just how many: a step that fails on errors alone cannot be judged from a count result["error_detail"] = [str(e)[:300] for e in new][:5] + # and when, for every counted record: error_detail keeps 300 characters of 5 records, which cut elapsed off 323 + # of the 353 records of r28 (the endpoint URIs come first), so a re-score had to guess + result["error_times"] = [record_times(e) for e in new] result.setdefault("errors_old_slice", result["errors"]) diff = list(difflib.unified_diff(before["route"].splitlines(), route.splitlines(), lineterm="", n=0)) diffp = list(difflib.unified_diff(before["props"].splitlines(), props.splitlines(), lineterm="", n=0)) @@ -854,14 +862,19 @@ def score(step, project, cfg, mcp, name, before, log=None): # began counts, and so do the ERROR records of the log newest = records_since(errs, before.get("last_write_sent") if FAST else None, max(last[1], before.get("error_count", 0))) - older = sum(1 for e in newest - if began_before(e, before.get("last_write_at"), before.get("last_write_spare", 0.5))) + old = [began_before(e, before.get("last_write_at"), before.get("last_write_spare", 0.5)) for e in newest] + older = sum(old) after += len(newest) - older + result["errors_after_last_write_times"] = [record_times(e) for e, o in zip(newest, old) if not o] + result["errors_older_version_times"] = [record_times(e) for e, o in zip(newest, old) if o] else: after = result["errors"] result["errors_after_last_write"] = after # how many errors after the last write were left out, as their exchange began before its reload result["errors_older_version"] = older + # when the last write was sent and when the version began (epoch seconds), and the seconds to spare of began_before + for k in ("last_write_sent", "last_write_at", "last_write_spare"): + result[k] = before.get(k) lastwrite_fine = after == 0 or bool(chk.get("errors_ok")) if chk.get("min_errors") and result["errors"] < chk["min_errors"]: lastwrite_fine = False
