branch: externals/llm
commit 46feff756f76b29ff544dc7936f20cd5499fc156
Author: Andrew Hyatt <[email protected]>
Commit: Andrew Hyatt <[email protected]>
Change request functionality to better handle streaming
Streaming does not yet exist in the llm package, this is the first change
to enable it.
---
NEWS.org | 2 ++
llm-request.el | 44 +++++++++++++++++++++++++++++++++-----------
2 files changed, 35 insertions(+), 11 deletions(-)
diff --git a/NEWS.org b/NEWS.org
index 484483d2fb..d25a09086b 100644
--- a/NEWS.org
+++ b/NEWS.org
@@ -1,2 +1,4 @@
+* Version 0.2.1
+- Changes in how we make and listen to requests, in preparation for streaming
functionality.
* Version 0.2
- Remove the dependency on non-GNU request library.
diff --git a/llm-request.el b/llm-request.el
index 2f57822c78..9a50beaf45 100644
--- a/llm-request.el
+++ b/llm-request.el
@@ -26,12 +26,19 @@
(defun llm-request--content ()
"From the current buffer, return the content of the response."
- (goto-char (point-min))
- (re-search-forward (rx (seq line-start
- (zero-or-one control)
- line-end)))
- (forward-line)
- (buffer-substring-no-properties (point) (point-max)))
+ (message "llm-request--content for buffer %s" (current-buffer))
+ (buffer-substring-no-properties
+ (or (and (boundp 'url-http-end-of-headers) url-http-end-of-headers)
+ (save-match-data
+ (save-excursion
+ (goto-char (point-min))
+ (search-forward "\n\n" nil t)
+ (forward-line)
+ (point))))
+ (point-max)))
+
+(defvar-local llm-request--partial-callback nil
+ "The callback to call when a partial response is received.")
(cl-defun llm-request-sync (url &key headers data timeout)
"Make a request to URL. The parsed response will be returned.
@@ -53,7 +60,15 @@ TIMEOUT is the number of seconds to wait for a response."
(json-read-from-string (llm-request--content)))
(error "LLM request timed out")))))
-(cl-defun llm-request-async (url &key headers data on-success on-error
on-chunk)
+(defun llm-request--handle-new-content (&rest _)
+ "Handle new content in the current buffer."
+ (save-match-data
+ (save-excursion
+ (with-auto-compression-mode
+ (when llm-request--partial-callback
+ (funcall llm-request--partial-callback (llm-request--content)))))))
+
+(cl-defun llm-request-async (url &key headers data on-success on-error
on-partial)
"Make a request to URL.
Nothing will be returned.
@@ -69,9 +84,13 @@ This is required.
ON-ERROR will be called with the error code and a response-body.
This is required.
-ON-CHUNK will be called with the potentially incomplete response
+ON-PARTIAL will be called with the potentially incomplete response
body as a string. This is an optional argument."
(let ((url-request-method "POST")
+ ;; This is necessary for streaming, otherwise we get gzip'd data that
is
+ ;; unparseable until the end. The responses should be small enough that
+ ;; this should not be any big loss.
+ (url-mime-encoding-string "identity")
(url-request-extra-headers
(append headers '(("Content-Type" . "application/json"))))
(url-request-data (json-encode data)))
@@ -80,6 +99,8 @@ body as a string. This is an optional argument."
url
;; For some reason the closure you'd expect did not work here.
(lambda (_ on-success on-error)
+ ;; No matter what, we need to stop listening for changes.
+ (remove-hook 'after-change-functions
#'llm-request--handle-new-content t)
(let ((code (url-http-parse-response)))
(if (eq code 200)
(funcall on-success (json-read-from-string
(llm-request--content)))
@@ -87,11 +108,12 @@ body as a string. This is an optional argument."
(json-read-from-string
(llm-request--content)))))))
(list on-success on-error)
t)))
- (when on-chunk
+ (when (and buffer on-partial)
(with-current-buffer buffer
+ (setq llm-request--partial-callback on-partial)
(add-hook 'after-change-functions
- (lambda (_ _ _)
- (funcall on-chunk (llm-request--content)))))))))
+ #'llm-request--handle-new-content
+ nil t))))))
(provide 'llm-request)
;;; llm-request.el ends here
