imbajin commented on code in PR #412:
URL: https://github.com/apache/hugegraph-ai/pull/412#discussion_r4578310176
##########
hugegraph-llm/src/hugegraph_llm/operators/index_op/vector_index_query.py:
##########
@@ -25,17 +25,45 @@
class VectorIndexQuery:
- def __init__(self, vector_index: type[VectorStoreBase], embedding:
BaseEmbedding, topk: int = 3):
+ """Retrieve matching chunks from the chunk vector index.
+
+ ``vector_dis_threshold`` is forwarded to ``VectorStoreBase.search`` and
+ defaults to the ``CHUNK_DIS_THRESHOLD`` setting (2.0, the historical
+ hard-coded value), which only prunes clearly-off chunk candidates;
+ keyword-to-vertex matching keeps its own ``VECTOR_DIS_THRESHOLD`` knob.
+ Ranking stays top-k and relevance is refined by the downstream
+ merge/rerank step.
+ """
+
+ def __init__(
+ self,
+ vector_index: type[VectorStoreBase],
+ embedding: BaseEmbedding,
+ topk: int = 3,
+ vector_dis_threshold: Optional[float] = None,
+ ):
self.embedding = embedding
self.topk = topk
+ self.vector_dis_threshold = (
+ huge_settings.chunk_dis_threshold if vector_dis_threshold is None
else vector_dis_threshold
+ )
self.vector_index =
vector_index.from_name(embedding.get_embedding_dim(), huge_settings.graph_name,
"chunks")
def run(self, context: Dict[str, Any]) -> Dict[str, Any]:
query = context.get("query")
query_embedding = self.embedding.get_texts_embeddings([query])[0]
- # TODO: why set dis_threshold=2?
- results = self.vector_index.search(query_embedding, self.topk,
dis_threshold=2)
- # TODO: check format results
- context["vector_result"] = results
- log.debug("KNOWLEDGE FROM VECTOR:\n%s", "\n".join(rel for rel in
context["vector_result"]))
+ results = self.vector_index.search(query_embedding, self.topk,
dis_threshold=self.vector_dis_threshold)
+ context["vector_result"] = self._check_results_format(results)
+ log.debug("KNOWLEDGE FROM VECTOR:\n%s",
"\n".join(context["vector_result"]))
return context
+
+ @staticmethod
+ def _check_results_format(results: List[Any]) -> List[str]:
+ """Drop non-string entries so downstream joins and reranks can rely on
the type."""
+ checked = []
+ for result in results:
+ if isinstance(result, str):
+ checked.append(result)
+ else:
+ log.warning("Filter out vector result with unexpected format:
%s", type(result).__name__)
Review Comment:
**⚠️ Preserve JSON-shaped text chunks from Milvus**
Milvus stores string chunks unchanged but decodes every retrieved property
with `json.loads`, so a valid chunk such as `123`, `true`, or a JSON object
arrives here as a number, boolean, or dictionary and is discarded. Previously
this could fail at the join; this change instead allows retrieval to complete
with missing context. Please preserve the original chunk text through the
Milvus round trip (including already indexed data), and cover JSON-shaped
chunks in a retrieval test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]