This is an automated email from the ASF dual-hosted git repository.
jin pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph-ai.git
The following commit(s) were added to refs/heads/main by this push:
new 6a897ee fix(llm): skip empty chunk in LLM steaming mode (#245)
6a897ee is described below
commit 6a897eea9397f6b9c06929861d5df3f3f2ecaad9
Author: SoJGooo <102796027+mrjs...@users.noreply.github.com>
AuthorDate: Thu May 22 15:40:39 2025 +0800
fix(llm): skip empty chunk in LLM steaming mode (#245)
This PR adds guards to skip over empty choices in streaming responses to
prevent errors during token generation.
- Adds an early continue when chunk.choices is empty in the synchronous
generate_streaming loop
- Adds the same guard in the asynchronous agenerate_streaming loop
---
Co-authored-by: imbajin <j...@apache.org>
---
README.md | 1 +
hugegraph-llm/README.md | 4 ++--
hugegraph-llm/pyproject.toml | 2 +-
hugegraph-llm/requirements.txt | 1 +
hugegraph-llm/src/hugegraph_llm/models/llms/litellm.py | 6 ++++++
hugegraph-llm/src/hugegraph_llm/models/llms/ollama.py | 3 +++
hugegraph-llm/src/hugegraph_llm/models/llms/openai.py | 8 +++++++-
7 files changed, 21 insertions(+), 4 deletions(-)
diff --git a/README.md b/README.md
index c0d57ae..638ddd4 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,7 @@
# hugegraph-ai
[](https://www.apache.org/licenses/LICENSE-2.0.html)
+[](https://deepwiki.com/apache/incubator-hugegraph-ai)
`hugegraph-ai` aims to explore the integration of
[HugeGraph](https://github.com/apache/hugegraph) with artificial
intelligence (AI) and provide comprehensive support for developers to leverage
HugeGraph's AI capabilities
diff --git a/hugegraph-llm/README.md b/hugegraph-llm/README.md
index 8e4d503..4c2a201 100644
--- a/hugegraph-llm/README.md
+++ b/hugegraph-llm/README.md
@@ -1,4 +1,4 @@
-# hugegraph-llm
+# hugegraph-llm [](https://deepwiki.com/apache/incubator-hugegraph-ai)
## 1. Summary
@@ -24,7 +24,7 @@ graph systems and large language models.
## 3. Preparation
1. Start the HugeGraph database, you can run it via
[Docker](https://hub.docker.com/r/hugegraph/hugegraph)/[Binary
Package](https://hugegraph.apache.org/docs/download/download/).
- There is a simple method by docker：
+ There is a simple method by docker:
```bash
docker run -itd --name=server -p 8080:8080 hugegraph/hugegraph
```
diff --git a/hugegraph-llm/pyproject.toml b/hugegraph-llm/pyproject.toml
index 467aa66..55e427a 100644
--- a/hugegraph-llm/pyproject.toml
+++ b/hugegraph-llm/pyproject.toml
@@ -88,4 +88,4 @@ include = [
allow-direct-references = true
[tool.uv.sources]
-hugegraph-python= {path = "../hugegraph-python-client/",editable = true}
+hugegraph-python = { path = "../hugegraph-python-client/", editable = true }
diff --git a/hugegraph-llm/requirements.txt b/hugegraph-llm/requirements.txt
index 3abe63e..5074a15 100644
--- a/hugegraph-llm/requirements.txt
+++ b/hugegraph-llm/requirements.txt
@@ -1,3 +1,4 @@
+# TODO: remove this file after uv run stable, don't update it
openai~=1.61.0
ollama~=0.4.8
qianfan~=0.3.18
diff --git a/hugegraph-llm/src/hugegraph_llm/models/llms/litellm.py
b/hugegraph-llm/src/hugegraph_llm/models/llms/litellm.py
index 7627979..b9cc0f1 100644
--- a/hugegraph-llm/src/hugegraph_llm/models/llms/litellm.py
+++ b/hugegraph-llm/src/hugegraph_llm/models/llms/litellm.py
@@ -128,6 +128,9 @@ class LiteLLMClient(BaseLLM):
)
result = ""
for chunk in response:
+ if not chunk.choices:
+ log.debug("Received empty choices in streaming chunk: %s",
chunk)
+ continue
if chunk.choices[0].delta.content:
result += chunk.choices[0].delta.content
if on_token_callback:
@@ -158,6 +161,9 @@ class LiteLLMClient(BaseLLM):
stream=True,
)
async for chunk in response:
+ if not chunk.choices:
+ log.debug("Received empty choices in streaming chunk: %s",
chunk)
+ continue
if chunk.choices[0].delta.content:
if on_token_callback:
on_token_callback(chunk)
diff --git a/hugegraph-llm/src/hugegraph_llm/models/llms/ollama.py
b/hugegraph-llm/src/hugegraph_llm/models/llms/ollama.py
index 58f063b..5354ba3 100644
--- a/hugegraph-llm/src/hugegraph_llm/models/llms/ollama.py
+++ b/hugegraph-llm/src/hugegraph_llm/models/llms/ollama.py
@@ -101,6 +101,9 @@ class OllamaClient(BaseLLM):
messages=messages,
stream=True
):
+ if not chunk["message"]:
+ log.debug("Received empty chunk['message'] in streaming chunk:
%s", chunk)
+ continue
token = chunk["message"]["content"]
if on_token_callback:
on_token_callback(token)
diff --git a/hugegraph-llm/src/hugegraph_llm/models/llms/openai.py
b/hugegraph-llm/src/hugegraph_llm/models/llms/openai.py
index b185314..88cea39 100644
--- a/hugegraph-llm/src/hugegraph_llm/models/llms/openai.py
+++ b/hugegraph-llm/src/hugegraph_llm/models/llms/openai.py
@@ -149,6 +149,9 @@ class OpenAIClient(BaseLLM):
)
for chunk in completions:
+ if not chunk.choices:
+ log.debug("Received empty choices in streaming chunk: %s",
chunk)
+ continue
delta = chunk.choices[0].delta
if delta.content:
token = delta.content
@@ -186,6 +189,9 @@ class OpenAIClient(BaseLLM):
stream=True
)
async for chunk in completions:
+ if not chunk.choices:
+ log.debug("Received empty choices in streaming chunk: %s",
chunk)
+ continue
delta = chunk.choices[0].delta
if delta.content:
token = delta.content
@@ -206,7 +212,7 @@ class OpenAIClient(BaseLLM):
raise e
def num_tokens_from_string(self, string: str) -> int:
- """Get token count from string."""
+ """Get token count from a string."""
encoding = tiktoken.encoding_for_model(self.model)
num_tokens = len(encoding.encode(string))
return num_tokens
