branch: externals/ellama
commit 3cdb58ed9b02c2e0b8b680f178cd548251279eb9
Author: Sergey Kostyaev <[email protected]>
Commit: Sergey Kostyaev <[email protected]>
Stabilize agent loops around compaction
Keep repeated agent and subagent errors counted through compaction so a
server outage cannot reset into an endless continue loop. Record compaction
errors and stop recovery when compaction fails instead of sending another
continue request. Skip one immediate preflight compaction after successful
compaction so deferred plan-and-act callbacks can resume the loop. Add
regressions for failed compaction recovery and continuation after response
compaction.
---
ellama-tools.el | 29 ++++---
ellama.el | 61 +++++++++-----
tests/test-ellama-tools.el | 206 ++++++++++++++++++++++++++++++++++++++++++++-
3 files changed, 262 insertions(+), 34 deletions(-)
diff --git a/ellama-tools.el b/ellama-tools.el
index 30fe6b27c2..e620691df3 100644
--- a/ellama-tools.el
+++ b/ellama-tools.el
@@ -4277,6 +4277,7 @@ ERROR-CB is called on non-tool LLM errors to track
consecutive failures."
(ellama-tools--make-agent-error-callback session)))))
(declare-function ellama--session-compact "ellama" (session &rest args))
+(declare-function ellama--session-extra-get "ellama" (session key))
(defun ellama-tools--agent-error-message (err)
"Return human-readable message for agent loop ERR."
@@ -4299,10 +4300,14 @@ ERROR-CB is called on non-tool LLM errors to track
consecutive failures."
((continue ()
(unless continued
(setq continued t)
- (ellama-tools--continue-agent-after-error session))))
- (unless (ellama--session-compact
- session :automatic t :on-done #'continue)
- (continue)))))
+ (if (ellama--session-extra-get
+ session :auto-compact-last-error)
+ (message "Agent loop stopped after compaction error: %s"
+ (ellama--session-extra-get
+ session :auto-compact-last-error))
+ (ellama-tools--continue-agent-after-error session)))))
+ (ellama--session-compact
+ session :automatic t :on-done #'continue))))
(defun ellama-tools--make-agent-error-callback (session)
"Return error callback for plan-and-act loop of SESSION.
@@ -4323,7 +4328,7 @@ and restarts the loop."
(message "Compacting session after repeated errors...")
(setq state
(ellama-tools--agent-state-put
- state :consecutive-error-count 0))
+ state :consecutive-error-count new-count))
(ellama-tools--agent-put-state session state)
(ellama-tools--compact-and-continue-agent session))
(setq state
@@ -4685,10 +4690,14 @@ ERROR-CB is called on non-tool LLM errors to track
consecutive failures."
((continue ()
(unless continued
(setq continued t)
- (ellama-tools--continue-subagent-after-error session))))
- (unless (ellama--session-compact
- session :automatic t :on-done #'continue)
- (continue)))))
+ (if (ellama--session-extra-get
+ session :auto-compact-last-error)
+ (message "Subagent stopped after compaction error: %s"
+ (ellama--session-extra-get
+ session :auto-compact-last-error))
+ (ellama-tools--continue-subagent-after-error session)))))
+ (ellama--session-compact
+ session :automatic t :on-done #'continue))))
(defun ellama-tools--make-subagent-error-callback (session)
"Return error callback for subagent loop of SESSION.
@@ -4706,7 +4715,7 @@ and restarts the loop."
(progn
(message "Compacting subagent session after repeated errors...")
(ellama-tools--set-session-extra
- session (plist-put extra :consecutive-error-count 0))
+ session (plist-put extra :consecutive-error-count new-count))
(ellama-tools--compact-and-continue-subagent session))
(ellama-tools--set-session-extra
session (plist-put extra :consecutive-error-count new-count))
diff --git a/ellama.el b/ellama.el
index c34b1eb00f..038210c794 100644
--- a/ellama.el
+++ b/ellama.el
@@ -1726,6 +1726,7 @@ REQUESTED-KEPT-TURNS is the configured recent turn count."
(defun ellama--session-compact-handle-async-error (session err)
"Reset SESSION compaction state and report asynchronous ERR."
+ (ellama--session-extra-put session :auto-compact-last-error err)
(ellama--session-compact-reset session nil)
(message "Ellama context compaction failed: %s" err))
@@ -1823,14 +1824,22 @@ If AUTOMATIC is non-nil, fail quietly and return nil."
(lambda (response)
(unless completed
(condition-case compact-err
- (ellama--session-compact-apply
- session prompt provider buffer original-context
- interactions old-interactions
recent-interactions
- before-token-count
- (if (stringp response)
- response
- (plist-get response :text))
- requested-keep-turns)
+ (progn
+ (ellama--session-extra-put
+ session :auto-compact-last-error nil)
+ (prog1
+ (ellama--session-compact-apply
+ session prompt provider buffer
+ original-context
+ interactions old-interactions
+ recent-interactions before-token-count
+ (if (stringp response)
+ response
+ (plist-get response :text))
+ requested-keep-turns)
+ (ellama--session-extra-put
+ session
+ :auto-compact-skip-next-preflight t)))
(error
(ellama--session-compact-handle-async-error
session (error-message-string compact-err))))
@@ -1865,6 +1874,8 @@ If AUTOMATIC is non-nil, fail quietly and return nil."
(error
(if automatic
(progn
+ (ellama--session-extra-put
+ session :auto-compact-last-error (error-message-string err))
(message "Ellama context compaction failed: %s"
(error-message-string err))
nil)
@@ -1899,20 +1910,26 @@ REQUEST-CONTEXT is the active request context."
(llm-chat-prompt-p prompt)
(not (ellama--session-extra-get
session :auto-compact-in-progress)))
- (when-let* ((token-count
- (ellama--session-compact-estimate-prompt-tokens
- provider prompt))
- (threshold
- (ellama--session-auto-compact-threshold provider))
- ((>= token-count threshold)))
- (ellama--session-compact
- session
- :provider provider
- :buffer buffer
- :token-count token-count
- :automatic t
- :on-done on-done
- :request-context request-context))))
+ (if (ellama--session-extra-get
+ session :auto-compact-skip-next-preflight)
+ (progn
+ (ellama--session-extra-put
+ session :auto-compact-skip-next-preflight nil)
+ nil)
+ (when-let* ((token-count
+ (ellama--session-compact-estimate-prompt-tokens
+ provider prompt))
+ (threshold
+ (ellama--session-auto-compact-threshold provider))
+ ((>= token-count threshold)))
+ (ellama--session-compact
+ session
+ :provider provider
+ :buffer buffer
+ :token-count token-count
+ :automatic t
+ :on-done on-done
+ :request-context request-context)))))
(defun ellama--active-session-by-id (id)
"Return active session matching display ID."
diff --git a/tests/test-ellama-tools.el b/tests/test-ellama-tools.el
index a2ab73e0d6..efa1fadebf 100644
--- a/tests/test-ellama-tools.el
+++ b/tests/test-ellama-tools.el
@@ -3159,17 +3159,74 @@ Return list with result and prompt."
(funcall callback "temporary failure again")
(should (= (plist-get (ellama-tools--agent-state session)
:consecutive-error-count)
- 0))
+ 2))
(should (= compact-count 1))
(should (= (length stream-calls) 1))
(should (functionp compact-done))
(funcall compact-done)
(should (= (length stream-calls) 2))
+ (funcall (ellama-tools--make-agent-loop-handler
+ session buffer "System")
+ "Recovered response")
+ (should (= (plist-get (ellama-tools--agent-state session)
+ :consecutive-error-count)
+ 0))
(should (string-match-p "Current plan state"
(caar stream-calls))))))
(when (buffer-live-p buffer)
(kill-buffer buffer)))))
+(ert-deftest test-ellama-agent-error-callback-stops-when-compact-fails ()
+ (ellama-test--ensure-local-ellama-tools)
+ (let* ((buffer (generate-new-buffer " *ellama-agent-compact-error-test*"))
+ (session
+ (make-ellama-session
+ :id "agent-compact-error"
+ :extra (list :uid "agent-compact-error-uid"
+ :tools nil
+ :agent-loop
+ (list :phase 'acting
+ :plan (list (list :id 1
+ :title "Keep working"
+ :status 'pending))
+ :step-count 0
+ :consecutive-error-count 1
+ :max-steps 5
+ :completed nil
+ :system "System"))))
+ (stream-calls nil)
+ (compact-count 0))
+ (unwind-protect
+ (progn
+ (with-current-buffer buffer
+ (org-mode))
+ (cl-letf (((symbol-function 'ellama-get-session-buffer)
+ (lambda (id)
+ (and (member id '("agent-compact-error"
+ "agent-compact-error-uid"))
+ buffer)))
+ ((symbol-function 'ellama-stream)
+ (lambda (prompt &rest args)
+ (push (list prompt args) stream-calls)))
+ ((symbol-function 'ellama--session-compact)
+ (lambda (compact-session &rest _args)
+ (setq compact-count (1+ compact-count))
+ (ellama--session-extra-put
+ compact-session :auto-compact-last-error
+ "summary server unavailable")
+ nil))
+ ((symbol-function 'message)
+ (lambda (&rest _args) nil)))
+ (funcall (ellama-tools--make-agent-error-callback session)
+ "temporary failure again")
+ (should (= compact-count 1))
+ (should-not stream-calls)
+ (should (= (plist-get (ellama-tools--agent-state session)
+ :consecutive-error-count)
+ 2))))
+ (when (buffer-live-p buffer)
+ (kill-buffer buffer)))))
+
(ert-deftest
test-ellama-agent-loop-handler-parses-fallback-state-and-continues ()
(ellama-test--ensure-local-ellama-tools)
(let* ((buffer (generate-new-buffer " *ellama-agent-loop-test*"))
@@ -3330,6 +3387,97 @@ END_ELLAMA_AGENT_STATE"))
(when (buffer-live-p callback-buffer)
(kill-buffer callback-buffer)))))
+(ert-deftest
test-ellama-agent-loop-handler-continues-after-response-compaction ()
+ (ellama-test--ensure-local-ellama-tools)
+ (let* ((buffer
+ (generate-new-buffer " *ellama-agent-response-compact-test*"))
+ (provider (make-llm-fake))
+ (prompt (llm-make-chat-prompt "user 1" :context "System"))
+ (base-tool (llm-make-tool :name "read_file" :function #'ignore))
+ (session
+ (make-ellama-session
+ :id "agent-response-compact"
+ :provider provider
+ :prompt prompt
+ :extra (list :uid "agent-response-compact-uid"
+ :tools (list base-tool)
+ :agent-loop
+ (list :phase 'acting
+ :plan (list (list :id 1
+ :title "Keep going"
+ :status 'pending))
+ :step-count 0
+ :max-steps 3
+ :completed nil
+ :system "System"))))
+ (ellama-response-process-method 'async)
+ (ellama-spinner-enabled nil)
+ (ellama-session-auto-compact-enabled t)
+ (ellama-session-auto-compact-token-threshold 100)
+ (ellama-session-auto-compact-provider (make-llm-fake))
+ (ellama-session-auto-compact-keep-last-turns 2)
+ (ellama-session-auto-compact-show-message nil)
+ (ellama-session-hide-org-quotes nil)
+ (call-count 0)
+ compact-callback
+ continuation-prompt)
+ (llm-chat-prompt-append-response prompt "assistant 1" 'assistant)
+ (llm-chat-prompt-append-response prompt "user 2")
+ (llm-chat-prompt-append-response prompt "assistant 2" 'assistant)
+ (llm-chat-prompt-append-response prompt "user 3")
+ (llm-chat-prompt-append-response prompt "assistant 3" 'assistant)
+ (llm-chat-prompt-append-response prompt "user 4")
+ (llm-chat-prompt-append-response prompt "assistant 4" 'assistant)
+ (unwind-protect
+ (progn
+ (with-current-buffer buffer
+ (org-mode)
+ (setq-local ellama--current-session session)
+ (setq-local ellama--change-group (prepare-change-group))
+ (activate-change-group ellama--change-group))
+ (cl-letf (((symbol-function 'ellama-get-session-buffer)
+ (lambda (id)
+ (and (member id '("agent-response-compact"
+ "agent-response-compact-uid"))
+ buffer)))
+ ((symbol-function 'llm-count-tokens)
+ (lambda (_provider _text) 120))
+ ((symbol-function 'llm-chat-async)
+ (lambda (_provider sent-prompt response-callback
+ _error-callback
+ &optional _multi-output)
+ (setq call-count (1+ call-count))
+ (if (= call-count 1)
+ (progn
+ (setq compact-callback response-callback)
+ 'compact-request)
+ (setq continuation-prompt sent-prompt)
+ 'continuation-request))))
+ (with-current-buffer buffer
+ (funcall
+ (ellama--response-handler
+ #'ignore nil buffer
+ (ellama-tools--make-agent-loop-handler
+ session buffer "System")
+ #'ignore provider prompt t #'identity)
+ '(:text "Controller response"
+ :input-tokens 90
+ :output-tokens 20)))
+ (should (= call-count 1))
+ (should compact-callback)
+ (should-not continuation-prompt)
+ (funcall compact-callback '(:text "Summary"))
+ (should (= call-count 2))
+ (should (llm-chat-prompt-p continuation-prompt))
+ (should (string-match-p
+ "Current plan state"
+ (llm-chat-prompt-to-text continuation-prompt)))
+ (should (string-match-p
+ "Next pending item: Keep going"
+ (llm-chat-prompt-to-text continuation-prompt)))))
+ (when (buffer-live-p buffer)
+ (kill-buffer buffer)))))
+
(ert-deftest test-ellama-agent-report-result-restores-original-tools ()
(ellama-test--ensure-local-ellama-tools)
(let* ((buffer (generate-new-buffer " *ellama-agent-done-test*"))
@@ -3527,17 +3675,71 @@ END_ELLAMA_AGENT_STATE"))
(funcall callback "temporary failure again")
(should (= (plist-get (ellama-session-extra session)
:consecutive-error-count)
- 0))
+ 2))
(should (= compact-count 1))
(should (= (length stream-calls) 1))
(should (functionp compact-done))
(funcall compact-done)
(should (= (length stream-calls) 2))
+ (funcall (ellama-tools--make-subagent-loop-handler
+ session worker-buffer "System")
+ "Recovered response")
+ (should (= (plist-get (ellama-session-extra session)
+ :consecutive-error-count)
+ 0))
(should (equal (caar stream-calls)
ellama-tools-subagent-continue-prompt)))))
(when (buffer-live-p worker-buffer)
(kill-buffer worker-buffer)))))
+(ert-deftest test-ellama-subagent-error-callback-stops-when-compact-fails ()
+ (ellama-test--ensure-local-ellama-tools)
+ (let* ((worker-buffer
+ (generate-new-buffer " *ellama-worker-compact-error-test*"))
+ (session
+ (make-ellama-session
+ :id "worker-compact-error"
+ :extra (list :uid "worker-compact-error-uid"
+ :task-completed nil
+ :step-count 0
+ :consecutive-error-count 1
+ :max-steps 5
+ :tools nil
+ :system "System"
+ :result-callback #'ignore)))
+ (stream-calls nil)
+ (compact-count 0))
+ (unwind-protect
+ (progn
+ (with-current-buffer worker-buffer
+ (org-mode))
+ (cl-letf (((symbol-function 'ellama-get-session-buffer)
+ (lambda (id)
+ (and (member id '("worker-compact-error"
+ "worker-compact-error-uid"))
+ worker-buffer)))
+ ((symbol-function 'ellama-stream)
+ (lambda (prompt &rest args)
+ (push (list prompt args) stream-calls)))
+ ((symbol-function 'ellama--session-compact)
+ (lambda (compact-session &rest _args)
+ (setq compact-count (1+ compact-count))
+ (ellama--session-extra-put
+ compact-session :auto-compact-last-error
+ "summary server unavailable")
+ nil))
+ ((symbol-function 'message)
+ (lambda (&rest _args) nil)))
+ (funcall (ellama-tools--make-subagent-error-callback session)
+ "temporary failure again")
+ (should (= compact-count 1))
+ (should-not stream-calls)
+ (should (= (plist-get (ellama-session-extra session)
+ :consecutive-error-count)
+ 2))))
+ (when (buffer-live-p worker-buffer)
+ (kill-buffer worker-buffer)))))
+
(ert-deftest test-ellama-subagent-loop-handler-uses-current-session-buffer ()
(ellama-test--ensure-local-ellama-tools)
(let* ((stale-buffer (generate-new-buffer " *ellama-worker-stale-test*"))
