branch: externals/ellama
commit 1a4cfe368b6cdaef6a33c8739782f7188a81c24e
Author: Sergey Kostyaev <[email protected]>
Commit: Sergey Kostyaev <[email protected]>
Keep agent loops running after LLM errors
Handle non-tool LLM failures in plan-and-act and subagent loops by
continuing after the first error, compacting after repeated errors, and
resuming once compaction completes.
Propagate agent error callbacks through plan-and-act entry points, clear
request state before invoking error callbacks, and add regression coverage for
both agent loop recovery paths.
---
ellama-tools.el | 99 +++++++++++++++++++++++++++---------
ellama.el | 8 ++-
tests/test-ellama-tools.el | 123 +++++++++++++++++++++++++++++++++++++++++++++
tests/test-ellama.el | 24 +++++++++
4 files changed, 229 insertions(+), 25 deletions(-)
diff --git a/ellama-tools.el b/ellama-tools.el
index 73853178aa..7604531260 100644
--- a/ellama-tools.el
+++ b/ellama-tools.el
@@ -4083,6 +4083,12 @@ ERROR-CB is called on non-tool LLM errors to track
consecutive failures."
(when parsed
(ellama-tools--agent-insert-state
session buffer (if (plist-get parsed :plan) "Plan" "Status")))
+ (when (and (stringp text)
+ (plist-get state :consecutive-error-count))
+ (setq state
+ (ellama-tools--agent-state-put
+ state :consecutive-error-count 0))
+ (ellama-tools--agent-put-state session state))
(cond
((not (ellama-session-p session))
(message "Plan-and-act session is not available."))
@@ -4138,7 +4144,34 @@ ERROR-CB is called on non-tool LLM errors to track
consecutive failures."
text session buffer system
(ellama-tools--make-agent-error-callback session)))))
-(declare-function ellama--session-compact "ellama" (session &key automatic))
+(declare-function ellama--session-compact "ellama"
+ (session &key automatic on-done))
+
+(defun ellama-tools--agent-error-message (err)
+ "Return human-readable message for agent loop ERR."
+ (cond
+ ((stringp err) err)
+ ((and (consp err) (symbolp (car err)))
+ (error-message-string err))
+ (t (format "%s" err))))
+
+(defun ellama-tools--continue-agent-after-error (session)
+ "Continue plan-and-act SESSION after an LLM error."
+ (ellama-tools--agent-loop-handler
+ nil session nil nil
+ (ellama-tools--make-agent-error-callback session)))
+
+(defun ellama-tools--compact-and-continue-agent (session)
+ "Compact plan-and-act SESSION and continue the loop."
+ (let (continued)
+ (cl-labels
+ ((continue ()
+ (unless continued
+ (setq continued t)
+ (ellama-tools--continue-agent-after-error session))))
+ (unless (ellama--session-compact
+ session :automatic t :on-done #'continue)
+ (continue)))))
(defun ellama-tools--make-agent-error-callback (session)
"Return error callback for plan-and-act loop of SESSION.
@@ -4147,26 +4180,26 @@ and restarts the loop."
(let ((session session))
(lambda (err)
(let* ((extra (ellama-session-extra session))
- (state (plist-get extra :agent-loop)))
+ (state (plist-get extra :agent-loop))
+ (message (ellama-tools--agent-error-message err)))
(unless (and state (plist-get state :completed))
(let* ((count (or (plist-get state :consecutive-error-count) 0))
(new-count (1+ count)))
(message "Agent loop error: %s (consecutive: %d)"
- (error-message-string err) new-count)
+ message new-count)
(if (>= new-count 2)
(progn
(message "Compacting session after repeated errors...")
- (ellama--session-compact session :automatic t)
- (setq state (ellama-tools--agent-state-put state
:consecutive-error-count 0))
+ (setq state
+ (ellama-tools--agent-state-put
+ state :consecutive-error-count 0))
(ellama-tools--agent-put-state session state)
- (ellama-tools--agent-loop-handler
- nil session nil nil
- (ellama-tools--make-agent-error-callback session)))
- (setq state (ellama-tools--agent-state-put state
:consecutive-error-count new-count))
+ (ellama-tools--compact-and-continue-agent session))
+ (setq state
+ (ellama-tools--agent-state-put
+ state :consecutive-error-count new-count))
(ellama-tools--agent-put-state session state)
- (ellama-tools--agent-loop-handler
- nil session nil nil
- (ellama-tools--make-agent-error-callback session)))))))))
+ (ellama-tools--continue-agent-after-error session))))))))
(defun ellama-tools-start-plan-and-act
(session buffer prompt &optional system tools max-steps)
@@ -4204,6 +4237,7 @@ automatic continuations."
buffer "Status" "Planning started.")
(list :system agent-system
:tools combined-tools
+ :on-error (ellama-tools--make-agent-error-callback session)
:on-done (ellama-tools--make-agent-loop-handler
session buffer agent-system))))
@@ -4229,6 +4263,7 @@ SYSTEM and TOOLS can override the stored runtime values."
(ellama-tools--session-extra-with session :tools combined-tools))
(list :system agent-system
:tools combined-tools
+ :on-error (ellama-tools--make-agent-error-callback session)
:on-done (ellama-tools--make-agent-loop-handler
session buffer agent-system))))
@@ -4424,8 +4459,8 @@ When the task is COMPLETE you MUST call `report_result`
exactly once."
(and (buffer-live-p session-buffer) session-buffer))
(and (buffer-live-p buffer) buffer)))
-(defun ellama--subagent-loop-handler (_text &optional session buffer system
error-cb)
- "Continue subagent SESSION loop in BUFFER with SYSTEM message.
+(defun ellama--subagent-loop-handler (text &optional session buffer system
error-cb)
+ "Continue subagent SESSION loop after TEXT in BUFFER with SYSTEM message.
ERROR-CB is called on non-tool LLM errors to track consecutive failures."
(let* ((session (or session ellama--current-session))
(extra (and (ellama-session-p session)
@@ -4438,6 +4473,10 @@ ERROR-CB is called on non-tool LLM errors to track
consecutive failures."
(tools (plist-get extra :tools))
(system (or system (plist-get extra :system)))
(buffer (ellama-tools--subagent-buffer session buffer)))
+ (when (and (stringp text)
+ (plist-get extra :consecutive-error-count))
+ (setq extra (plist-put extra :consecutive-error-count 0))
+ (ellama-tools--set-session-extra session extra))
(cond
((not (ellama-session-p session))
(message "Subagent session is not available."))
@@ -4479,6 +4518,24 @@ ERROR-CB is called on non-tool LLM errors to track
consecutive failures."
text session buffer system
(ellama-tools--make-subagent-error-callback session)))))
+(defun ellama-tools--continue-subagent-after-error (session)
+ "Continue subagent SESSION after an LLM error."
+ (ellama--subagent-loop-handler
+ nil session nil nil
+ (ellama-tools--make-subagent-error-callback session)))
+
+(defun ellama-tools--compact-and-continue-subagent (session)
+ "Compact subagent SESSION and continue the loop."
+ (let (continued)
+ (cl-labels
+ ((continue ()
+ (unless continued
+ (setq continued t)
+ (ellama-tools--continue-subagent-after-error session))))
+ (unless (ellama--session-compact
+ session :automatic t :on-done #'continue)
+ (continue)))))
+
(defun ellama-tools--make-subagent-error-callback (session)
"Return error callback for subagent loop of SESSION.
Increments consecutive-error-count; if it reaches 2, compacts the session
@@ -4487,23 +4544,19 @@ and restarts the loop."
(lambda (err)
(let* ((extra (ellama-session-extra session))
(count (or (plist-get extra :consecutive-error-count) 0))
- (new-count (1+ count)))
+ (new-count (1+ count))
+ (message (ellama-tools--agent-error-message err)))
(message "Subagent error: %s (consecutive: %d)"
- (error-message-string err) new-count)
+ message new-count)
(if (>= new-count 2)
(progn
(message "Compacting subagent session after repeated errors...")
- (ellama--session-compact session :automatic t)
(ellama-tools--set-session-extra
session (plist-put extra :consecutive-error-count 0))
- (ellama--subagent-loop-handler
- nil session nil nil
- (ellama-tools--make-subagent-error-callback session)))
+ (ellama-tools--compact-and-continue-subagent session))
(ellama-tools--set-session-extra
session (plist-put extra :consecutive-error-count new-count))
- (ellama--subagent-loop-handler
- nil session nil nil
- (ellama-tools--make-subagent-error-callback session)))))))
+ (ellama-tools--continue-subagent-after-error session))))))
(defun ellama-tools-task-tool
(callback &optional description role template template-base arguments)
diff --git a/ellama.el b/ellama.el
index d0d5dc6091..db570a00e8 100644
--- a/ellama.el
+++ b/ellama.el
@@ -3235,8 +3235,8 @@ REQUEST-CONTEXT is request context."
(accept-change-group ellama--change-group))
(when ellama-spinner-enabled
(spinner-stop))
- (funcall errcb msg)
- (ellama--deactivate-current-request request-context)))))
+ (ellama--deactivate-current-request request-context)
+ (funcall errcb msg)))))
(defun ellama--run-done-callback (donecb text)
"Call DONECB with TEXT."
@@ -4142,6 +4142,7 @@ after compaction."
max-steps)))
(agent-system (plist-get agent :system))
(agent-tools (plist-get agent :tools))
+ (errcb (plist-get agent :on-error))
(donecb (plist-get agent :on-done)))
(ellama-stream
prompt
@@ -4149,6 +4150,7 @@ after compaction."
:system agent-system
:tools agent-tools
:max-tokens max-tokens
+ :on-error errcb
:on-done donecb
:filter (when (derived-mode-p 'org-mode)
#'ellama--translate-markdown-to-org-filter)))))))
@@ -4191,12 +4193,14 @@ after compaction."
session (current-buffer))))
(system (plist-get agent :system))
(tools (plist-get agent :tools))
+ (errcb (plist-get agent :on-error))
(donecb (or (plist-get agent :on-done)
#'ellama-chat-done)))
(ellama-stream text
:session session
:system system
:tools tools
+ :on-error errcb
:on-done donecb
:filter (when (derived-mode-p 'org-mode)
#'ellama--translate-markdown-to-org-filter)))))
diff --git a/tests/test-ellama-tools.el b/tests/test-ellama-tools.el
index b2ba8fe8b9..596ed0219d 100644
--- a/tests/test-ellama-tools.el
+++ b/tests/test-ellama-tools.el
@@ -2881,6 +2881,7 @@ Return list with result and prompt."
(plist-get
(ellama-tools--agent-state session)
:system)))
+ (should (functionp (plist-get result :on-error)))
(should (functionp (plist-get result :on-done)))
(should (equal (plist-get
(ellama-tools--agent-state session)
@@ -2906,6 +2907,68 @@ Return list with result and prompt."
(when (buffer-live-p buffer)
(kill-buffer buffer)))))
+(ert-deftest test-ellama-agent-error-callback-continues-and-compacts-on-repeat
()
+ (ellama-test--ensure-local-ellama-tools)
+ (let* ((buffer (generate-new-buffer " *ellama-agent-error-test*"))
+ (base-tool (llm-make-tool :name "read_file" :function #'ignore))
+ (session
+ (make-ellama-session
+ :id "agent-error"
+ :extra (list :uid "agent-error-uid"
+ :tools (list base-tool)
+ :agent-loop
+ (list :phase 'acting
+ :plan (list (list :id 1
+ :title "Keep working"
+ :status 'pending))
+ :step-count 0
+ :consecutive-error-count 0
+ :max-steps 5
+ :completed nil
+ :system "System"))))
+ (stream-calls nil)
+ (compact-count 0)
+ (compact-done nil))
+ (unwind-protect
+ (progn
+ (with-current-buffer buffer
+ (org-mode))
+ (cl-letf (((symbol-function 'ellama-get-session-buffer)
+ (lambda (id)
+ (and (member id '("agent-error"
+ "agent-error-uid"))
+ buffer)))
+ ((symbol-function 'ellama-stream)
+ (lambda (prompt &rest args)
+ (push (list prompt args) stream-calls)))
+ ((symbol-function 'ellama--session-compact)
+ (lambda (_session &rest args)
+ (setq compact-count (1+ compact-count))
+ (setq compact-done (plist-get args :on-done))
+ t))
+ ((symbol-function 'message)
+ (lambda (&rest _args) nil)))
+ (let ((callback (ellama-tools--make-agent-error-callback session)))
+ (funcall callback "temporary failure")
+ (should (= (plist-get (ellama-tools--agent-state session)
+ :consecutive-error-count)
+ 1))
+ (should (= (length stream-calls) 1))
+ (should (= compact-count 0))
+ (funcall callback "temporary failure again")
+ (should (= (plist-get (ellama-tools--agent-state session)
+ :consecutive-error-count)
+ 0))
+ (should (= compact-count 1))
+ (should (= (length stream-calls) 1))
+ (should (functionp compact-done))
+ (funcall compact-done)
+ (should (= (length stream-calls) 2))
+ (should (string-match-p "Current plan state"
+ (caar stream-calls))))))
+ (when (buffer-live-p buffer)
+ (kill-buffer buffer)))))
+
(ert-deftest
test-ellama-agent-loop-handler-parses-fallback-state-and-continues ()
(ellama-test--ensure-local-ellama-tools)
(let* ((buffer (generate-new-buffer " *ellama-agent-loop-test*"))
@@ -2956,6 +3019,7 @@ END_ELLAMA_AGENT_STATE"))
session))
(should (equal (plist-get (cadr stream-call) :tools)
(list base-tool)))
+ (should (functionp (plist-get (cadr stream-call) :on-error)))
(should (functionp (plist-get (cadr stream-call) :on-done)))
(should (string-match-p "Current plan state"
(car stream-call)))
@@ -3119,6 +3183,65 @@ END_ELLAMA_AGENT_STATE"))
(when (buffer-live-p worker-buffer)
(kill-buffer worker-buffer)))))
+(ert-deftest
test-ellama-subagent-error-callback-continues-and-compacts-on-repeat ()
+ (ellama-test--ensure-local-ellama-tools)
+ (let* ((worker-buffer (generate-new-buffer " *ellama-worker-error-test*"))
+ (role-tool (llm-make-tool :name "read_file" :function #'ignore))
+ (session
+ (make-ellama-session
+ :id "worker-error"
+ :extra (list :uid "worker-error-uid"
+ :task-completed nil
+ :step-count 0
+ :consecutive-error-count 0
+ :max-steps 5
+ :tools (list role-tool)
+ :system "System"
+ :result-callback #'ignore)))
+ (stream-calls nil)
+ (compact-count 0)
+ (compact-done nil))
+ (unwind-protect
+ (progn
+ (with-current-buffer worker-buffer
+ (org-mode))
+ (cl-letf (((symbol-function 'ellama-get-session-buffer)
+ (lambda (id)
+ (and (member id '("worker-error"
+ "worker-error-uid"))
+ worker-buffer)))
+ ((symbol-function 'ellama-stream)
+ (lambda (prompt &rest args)
+ (push (list prompt args) stream-calls)))
+ ((symbol-function 'ellama--session-compact)
+ (lambda (_session &rest args)
+ (setq compact-count (1+ compact-count))
+ (setq compact-done (plist-get args :on-done))
+ t))
+ ((symbol-function 'message)
+ (lambda (&rest _args) nil)))
+ (let ((callback
+ (ellama-tools--make-subagent-error-callback session)))
+ (funcall callback "temporary failure")
+ (should (= (plist-get (ellama-session-extra session)
+ :consecutive-error-count)
+ 1))
+ (should (= (length stream-calls) 1))
+ (should (= compact-count 0))
+ (funcall callback "temporary failure again")
+ (should (= (plist-get (ellama-session-extra session)
+ :consecutive-error-count)
+ 0))
+ (should (= compact-count 1))
+ (should (= (length stream-calls) 1))
+ (should (functionp compact-done))
+ (funcall compact-done)
+ (should (= (length stream-calls) 2))
+ (should (equal (caar stream-calls)
+ ellama-tools-subagent-continue-prompt)))))
+ (when (buffer-live-p worker-buffer)
+ (kill-buffer worker-buffer)))))
+
(ert-deftest test-ellama-subagent-loop-handler-uses-current-session-buffer ()
(ellama-test--ensure-local-ellama-tools)
(let* ((stale-buffer (generate-new-buffer " *ellama-worker-stale-test*"))
diff --git a/tests/test-ellama.el b/tests/test-ellama.el
index 85840c5853..68c30107f0 100644
--- a/tests/test-ellama.el
+++ b/tests/test-ellama.el
@@ -1075,6 +1075,7 @@ detailed comparison to help you decide:
(should (equal (car stream-call) "Do work"))
(should (eq (plist-get (cadr stream-call) :session)
session))
+ (should (functionp (plist-get (cadr stream-call) :on-error)))
(should (functionp (plist-get (cadr stream-call) :on-done)))
(should (string-match-p
"PLAN AND ACT INSTRUCTIONS"
@@ -1139,6 +1140,7 @@ detailed comparison to help you decide:
session))
(should (equal (plist-get (cadr stream-call) :system)
"Existing system"))
+ (should (functionp (plist-get (cadr stream-call) :on-error)))
(should (functionp (plist-get (cadr stream-call) :on-done)))
(should (equal (plist-get
(car (plist-get
@@ -1347,6 +1349,7 @@ detailed comparison to help you decide:
(should (eq (plist-get (cadr stream-call) :session) session))
(should (equal (plist-get (cadr stream-call) :system)
"Agent system"))
+ (should (functionp (plist-get (cadr stream-call) :on-error)))
(should (functionp (plist-get (cadr stream-call) :on-done)))
(should (not (eq (plist-get (cadr stream-call) :on-done)
#'ellama-chat-done)))
@@ -2052,6 +2055,27 @@ detailed comparison to help you decide:
(should (equal done-text "Recovered answer"))
(should (equal (buffer-string) "Recovered answer"))))))
+(ert-deftest test-ellama-error-handler-deactivates-before-on-error ()
+ (let ((ellama-spinner-enabled nil)
+ (ellama-undo-on-error nil)
+ (active-request-during-callback :unset))
+ (with-temp-buffer
+ (let* ((request-context (ellama--make-request-context
+ (list (current-buffer))))
+ (handler (ellama--error-handler
+ (current-buffer)
+ (lambda (_msg)
+ (setq active-request-during-callback
+ ellama--current-request))
+ nil nil request-context)))
+ (setq ellama--change-group (prepare-change-group))
+ (activate-change-group ellama--change-group)
+ (ellama--set-current-request
+ 'request (list (current-buffer)) request-context)
+ (funcall handler 'error "temporary failure")
+ (should (null active-request-during-callback))
+ (should (null ellama--current-request))))))
+
(ert-deftest test-ellama-normalize-tool-use-args-decodes-unibyte-json ()
(let* ((raw-json
(encode-coding-string
