leaves12138 commented on code in PR #62:
URL:
https://github.com/apache/paimon-vector-index/pull/62#discussion_r3651827691
##########
python/paimon_vindex/__init__.py:
##########
@@ -15,21 +15,70 @@
# specific language governing permissions and limitations
# under the License.
+import ctypes
+import operator
+import threading
from dataclasses import dataclass
+from enum import IntEnum
from typing import Mapping, Optional
-import ctypes
import numpy as np
from . import _ffi
from ._ffi import lib
+_SIZE_T_MAX = ctypes.c_size_t(-1).value
+_UINT64_MAX = ctypes.c_uint64(-1).value
+
+
+def _size_t(value, name: str, *, allow_zero: bool) -> int:
+ try:
+ value = operator.index(value)
+ except TypeError as exc:
+ raise ValueError(f"{name} must be an integer") from exc
+ lower_bound = 0 if allow_zero else 1
+ if not lower_bound <= value <= _SIZE_T_MAX:
+ raise ValueError(
+ f"{name} must be in [{lower_bound}, {_SIZE_T_MAX}]"
+ )
+ return value
+
+
+def _uint64(value, name: str) -> int:
+ try:
+ value = operator.index(value)
+ except TypeError as exc:
+ raise ValueError(f"{name} must be an integer") from exc
+ if not 0 <= value <= _UINT64_MAX:
+ raise ValueError(f"{name} must be in [0, {_UINT64_MAX}]")
+ return value
+
+
+class _NativeHandleLock:
+ """Serialize native handles while failing same-thread callback reentry."""
+
+ def __init__(self):
+ self._lock = threading.Lock()
+ self._local = threading.local()
+
+ def __enter__(self):
+ if getattr(self._local, "active", False):
Review Comment:
The same cross-thread callback reentry deadlock exists in the Python
wrapper. `_NativeHandleLock` uses thread-local state, so a Rayon worker
callback sees `active == false` and then blocks on the lock held by the caller
whose native batch search is waiting for that worker. I reproduced this with a
5,000-row DiskANN index, `RAYON_NUM_THREADS=4`, a resident budget only 8 KiB
above the required 75,254 bytes, and a 64-query batch. The control run
completed with 488 reads across the caller plus four worker threads; when the
first worker-thread `pread_many` callback called `reader.metadata()`, the
process timed out after ten seconds. Could callback reentry be rejected based
on the active native operation/callback context rather than per-thread local
state?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
