leaves12138 commented on code in PR #62:
URL:
https://github.com/apache/paimon-vector-index/pull/62#discussion_r3651309256
##########
python/paimon_vindex/__init__.py:
##########
@@ -401,15 +447,54 @@ def __del__(self):
class VectorIndexReader:
- def __init__(self, input):
+ def __init__(
+ self,
+ input,
+ storage_profile: StorageProfile = StorageProfile.AUTO,
+ memory_budget_bytes: int = 4 * 1024 * 1024 * 1024,
+ ):
self._input = input
self._closed = False
+ profile_names = {
+ "auto": StorageProfile.AUTO,
+ "memory": StorageProfile.MEMORY,
+ "local_storage": StorageProfile.LOCAL_STORAGE,
+ "remote_storage": StorageProfile.REMOTE_STORAGE,
+ "object_store": StorageProfile.OBJECT_STORE,
+ }
+ try:
+ storage_profile = profile_names.get(storage_profile,
storage_profile)
+ storage_profile = StorageProfile(storage_profile)
+ except ValueError as exc:
+ raise ValueError(f"invalid storage_profile: {storage_profile}")
from exc
+ if memory_budget_bytes < 0:
+ raise ValueError("memory_budget_bytes must be non-negative")
+
self._read_ranges_callback = _make_read_ranges_callback(self._input)
input_file = _ffi.PaimonVindexInputFile()
input_file.ctx = None
input_file.read_ranges_fn = self._read_ranges_callback
- self._handle = lib.paimon_vindex_reader_open(input_file)
+ capability_names = (
+ "preferred_alignment_bytes",
+ "preferred_window_bytes",
+ "max_ranges_per_read",
+ )
+ capabilities = {
+ name: int(getattr(self._input, name, 0)) for name in
capability_names
+ }
+ if any(value < 0 for value in capabilities.values()):
+ raise ValueError("input read capabilities must be non-negative")
+ input_file.preferred_alignment_bytes = capabilities[
+ "preferred_alignment_bytes"
+ ]
+ input_file.preferred_window_bytes =
capabilities["preferred_window_bytes"]
+ input_file.max_ranges_per_read = capabilities["max_ranges_per_read"]
+ options = _ffi.PaimonVindexReaderOptions(
+ int(storage_profile),
+ memory_budget_bytes,
+ )
+ self._handle = lib.paimon_vindex_reader_open_with_options(input_file,
options)
Review Comment:
The Python wrapper needs the same native-handle serialization that was added
to the Java wrapper. `ctypes.CDLL` releases the GIL during these calls, so two
Python threads can enter `paimon_vindex_reader_search` on the same handle; the
C ABI then creates two simultaneous `&mut VectorIndexReader` references. I
reproduced this with two `ThreadPoolExecutor` searches on one fresh DiskANN
reader (using a barrier in `pread_many`): by the second iteration one call hit
`native panic: attempt to multiply with overflow` in `pq.rs`, after concurrent
reader initialization/search corrupted the observed PQ state. `close()` can
race the same way and free an in-use handle. Could we guard every reader native
call plus `close()` with a per-instance lock (and do the same for the other
mutable handle wrappers)?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
