branch: externals/llm
commit 5379cc0023dc333618d1d6609d3661d0b5979783
Author: Andrew Hyatt <[email protected]>
Commit: Andrew Hyatt <[email protected]>
Fix encoding and decoding problems of double-wide, or wider, chars
---
NEWS.org | 1 +
llm-request.el | 23 ++++++++++++-----------
2 files changed, 13 insertions(+), 11 deletions(-)
diff --git a/NEWS.org b/NEWS.org
index a1194aeb4a..37f87403ff 100644
--- a/NEWS.org
+++ b/NEWS.org
@@ -1,5 +1,6 @@
* Version 0.3
- Streaming support in the API, and for the Open AI and Vertex models.
+- Properly encode and decode in utf-8 so double-width or other character sizes
don't cause problems.
* Version 0.2.1
- Changes in how we make and listen to requests, in preparation for streaming
functionality.
- Fix overzealous change hook creation when using async llm requests.
diff --git a/llm-request.el b/llm-request.el
index aa0640b91d..7eaf715db6 100644
--- a/llm-request.el
+++ b/llm-request.el
@@ -27,15 +27,16 @@
(defun llm-request--content ()
"From the current buffer, return the content of the response."
- (buffer-substring-no-properties
- (or (and (boundp 'url-http-end-of-headers) url-http-end-of-headers)
- (save-match-data
- (save-excursion
- (goto-char (point-min))
- (search-forward "\n\n" nil t)
- (forward-line)
- (point))))
- (point-max)))
+ (decode-coding-string
+ (buffer-substring-no-properties
+ (or (and (boundp 'url-http-end-of-headers) url-http-end-of-headers)
+ (save-match-data
+ (save-excursion
+ (goto-char (point-min))
+ (search-forward "\n\n" nil t)
+ (forward-line)
+ (point))))
+ (point-max)) 'utf-8))
(defvar-local llm-request--partial-callback nil
"The callback to call when a partial response is received.")
@@ -53,7 +54,7 @@ TIMEOUT is the number of seconds to wait for a response."
(let ((url-request-method "POST")
(url-request-extra-headers
(append headers '(("Content-Type" . "application/json"))))
- (url-request-data (json-encode data)))
+ (url-request-data (encode-coding-string (json-encode data) 'utf-8)))
(let ((buf (url-retrieve-synchronously url t nil (or timeout 5))))
(if buf
(with-current-buffer buf
@@ -99,7 +100,7 @@ the buffer is turned into JSON and passed to ON-SUCCESS."
(url-mime-encoding-string "identity")
(url-request-extra-headers
(append headers '(("Content-Type" . "application/json"))))
- (url-request-data (json-encode data)))
+ (url-request-data (encode-coding-string (json-encode data) 'utf-8)))
(let ((buffer
(url-retrieve
url
