This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm-ffi.git
The following commit(s) were added to refs/heads/main by this push:
new 9faeaff3 [Rust] Reorder object, error, array, macro and structural
engine files; inline dispatch handlers, cold visitor stop path (#946)
9faeaff3 is described below
commit 9faeaff3532b89746f6af841e616e1ec9b520871
Author: Tianqi Chen <[email protected]>
AuthorDate: Fri Nov 6 15:53:14 2026 -0500
[Rust] Reorder object, error, array, macro and structural engine files;
inline dispatch handlers, cold visitor stop path (#946)
Reorder items within each file so it reads top-down: key data structures
first, then the main flow, then helpers. `object.rs` leads with
`Object`, `ObjectArc` (with its impls, `ImmutableObject` and its
`Send`/`Sync` next to `Clone`) and `ObjectRef`, then the traits, then
`ObjectIdentity`/`is_instance_of`/`fmt_object_ref`, then a Helpers
section (the crate-private `root_*` helpers, `type_key_of_index`,
`write_object_cell`, `allocate_object`), then `unsafe_`. `error.rs`
leads with `ErrorObj`, `Error` and `Result`, then `ErrorKind` and its
constants, then `impl Error` with the private `make_mut` after the
public accessors, with the std impls at the end. `collections/array.rs`
puts the private `Array` methods after the public ones and groups the
rest into Index, Iterators, Any conversions and Helpers (`write_cell`)
sections. `macros.rs` moves `unsafe_impl_object_ref_upcast!` and
`unsafe_impl_object_ref_downcast_ref!` out of the error-handling section
into their own "Macros for object references" section.
`extra/structural_visit.rs` and `extra/structural_mutate.rs` now read:
data structures (each type with its impls), public API and entrypoint,
the engine (its two `impl` blocks joined), prefix operations, private
utilities, outlined errors. The reorder commit is moves only; review it
with `--color-moved`.
Three small follow-ups ride along. `#[dispatch]` marks handlers
`#[inline(always)]` unless they already carry an `inline` attribute, so
a traversal no longer pays an outlined handler frame per node (no-op
mutate 210 to 158 ns on a 17-node tree, matching map).
`StructuralVisitorObj::with_visit_context_raw` returns a None carrier
directly and sends the rest to a `#[cold]` `#[inline(never)]` helper, so
continuing visits keep no input copies for error annotation (native-hook
visits 7–9% faster, DAG walk/visit about 3.5%).
`tests/python/test_stubgen_rust.py` gets the ruff-format fix that `main`
has been failing since #945.
---
rust/tvm-ffi-macros/src/dispatch.rs | 18 +-
rust/tvm-ffi/src/collections/array.rs | 172 ++---
rust/tvm-ffi/src/error.rs | 124 ++--
rust/tvm-ffi/src/extra/structural_mutate.rs | 979 ++++++++++++++--------------
rust/tvm-ffi/src/extra/structural_visit.rs | 941 +++++++++++++-------------
rust/tvm-ffi/src/lib.rs | 3 +-
rust/tvm-ffi/src/macros.rs | 317 ++++-----
rust/tvm-ffi/src/object.rs | 614 ++++++++---------
tests/python/test_stubgen_rust.py | 4 +-
9 files changed, 1617 insertions(+), 1555 deletions(-)
diff --git a/rust/tvm-ffi-macros/src/dispatch.rs
b/rust/tvm-ffi-macros/src/dispatch.rs
index 2e9b4ff1..f2bbee50 100644
--- a/rust/tvm-ffi-macros/src/dispatch.rs
+++ b/rust/tvm-ffi-macros/src/dispatch.rs
@@ -29,7 +29,8 @@ use crate::utils::get_tvm_ffi_crate;
pub(crate) fn dispatch(attr: TokenStream, item: TokenStream) -> TokenStream {
let args = parse_macro_input!(attr as DispatchArgs);
- let item_impl = parse_macro_input!(item as ItemImpl);
+ let mut item_impl = parse_macro_input!(item as ItemImpl);
+ inline_handlers(&mut item_impl, &args);
match expand(&item_impl, &args) {
Ok(generated) => quote!(#item_impl #generated).into(),
Err(error) => {
@@ -40,6 +41,21 @@ pub(crate) fn dispatch(attr: TokenStream, item: TokenStream)
-> TokenStream {
}
}
+// Handlers sit on the hot path of every dispatched value, and outlining them
+// costs a frame per node, so inline them unless the user chose otherwise.
+fn inline_handlers(item: &mut ItemImpl, args: &DispatchArgs) {
+ let method_prefix = format!("{}_", args.prefix);
+ for member in &mut item.items {
+ if let ImplItem::Method(method) = member {
+ if method.sig.ident.to_string().starts_with(&method_prefix)
+ && !method.attrs.iter().any(|attr|
attr.path.is_ident("inline"))
+ {
+ method.attrs.push(syn::parse_quote!(#[inline(always)]));
+ }
+ }
+ }
+}
+
struct DispatchArgs {
prefix: syn::Ident,
arguments: syn::TypeTuple,
diff --git a/rust/tvm-ffi/src/collections/array.rs
b/rust/tvm-ffi/src/collections/array.rs
index 24539917..5d00e5aa 100644
--- a/rust/tvm-ffi/src/collections/array.rs
+++ b/rust/tvm-ffi/src/collections/array.rs
@@ -52,13 +52,6 @@ unsafe impl ObjectCoreWithExtraItems for ArrayObj {
}
}
-/// Move `value` into the uninitialized `cell`.
-unsafe fn write_cell<T: ContainerElement>(cell: *mut TVMFFIAny, value: T) {
- let mut raw = TVMFFIAny::new();
- T::container_move_to_any(value, &mut raw);
- std::ptr::write(cell, raw);
-}
-
// Runs only for a Rust-allocated array; one allocated in C++ is released by
its
// own deleter.
impl Drop for ArrayObj {
@@ -217,33 +210,6 @@ impl<T: ContainerElement + Clone> Array<T> {
Self::new_with_capacity(items, capacity)
}
- /// Internal helper to allocate an ArrayObj with specific headroom.
- fn new_with_capacity(items: Vec<T>, capacity: usize) -> Self {
- let size = items.len();
-
- // Allocate with capacity
- let mut arc = ObjectArc::<ArrayObj>::new_with_extra_items(ArrayObj {
- object: Object::new(),
- data: core::ptr::null_mut(),
- size: size as i64,
- capacity: capacity as i64,
- data_deleter: None,
- });
-
- unsafe {
- let container = &mut *ObjectArc::as_mut_ptr(&mut arc);
-
- let base_ptr = ArrayObj::extra_items_mut(container).as_mut_ptr();
- container.data = base_ptr as *mut _;
-
- for (i, item) in items.into_iter().enumerate() {
- write_cell(base_ptr.add(i), item);
- }
- }
- // SAFETY: `arc` was allocated and initialized above as an `Array<T>`.
- unsafe { Self::from_data(arc) }
- }
-
/// Append `value` to the end.
///
/// See also [`List::push`](crate::List::push).
@@ -345,50 +311,6 @@ impl<T: ContainerElement + Clone> Array<T> {
Ok(())
}
- /// Make this handle the only reference to its cells, with room for at
least
- /// `capacity` of them, and return the container to edit in place.
- fn make_mut(&mut self, capacity: usize) -> &mut ArrayObj {
- // As C++ `Array::CopyOnWrite(reserve)`.
- let unique = ObjectArc::is_unique(&self.data);
- let current = self.data.capacity as usize;
- if !unique || capacity > current {
- let capacity = if capacity > current {
- (current * 2).max(capacity)
- } else {
- current
- };
- self.copy_to_new_buffer(capacity, unique);
- }
- // SAFETY: this handle is now the only reference, so no one else
observes
- // the container while it is mutably borrowed.
- unsafe { &mut *ObjectArc::as_mut_ptr(&mut self.data) }
- }
-
- /// Replace `self` with a unique array of `capacity` cells holding the same
- /// elements: moved out of a `unique` source, otherwise each re-referenced.
- fn copy_to_new_buffer(&mut self, capacity: usize, unique: bool) {
- let size = self.len();
- let mut grown = Self::new_with_capacity(Vec::new(), capacity);
- let source = self.as_container().data as *const TVMFFIAny;
- // SAFETY: `grown` is fresh with room for `size` cells. A unique source
- // forgets the cells it moved; a shared one keeps its own references.
- unsafe {
- let target = ObjectArc::as_mut_ptr(&mut grown.data);
- let cells = (*target).data as *mut TVMFFIAny;
- if unique {
- std::ptr::copy_nonoverlapping(source, cells, size);
- (*ObjectArc::as_mut_ptr(&mut self.data)).size = 0;
- } else {
- for i in 0..size {
- let copy =
Any::from(AnyView::from_raw_ffi_any(*source.add(i)));
- std::ptr::write(cells.add(i), Any::into_raw_ffi_any(copy));
- }
- }
- (*target).size = size as i64;
- }
- *self = grown;
- }
-
/// Return the number of elements.
///
/// # Returns
@@ -442,6 +364,77 @@ impl<T: ContainerElement + Clone> Array<T> {
}
}
+ /// Internal helper to allocate an ArrayObj with specific headroom.
+ fn new_with_capacity(items: Vec<T>, capacity: usize) -> Self {
+ let size = items.len();
+
+ // Allocate with capacity
+ let mut arc = ObjectArc::<ArrayObj>::new_with_extra_items(ArrayObj {
+ object: Object::new(),
+ data: core::ptr::null_mut(),
+ size: size as i64,
+ capacity: capacity as i64,
+ data_deleter: None,
+ });
+
+ unsafe {
+ let container = &mut *ObjectArc::as_mut_ptr(&mut arc);
+
+ let base_ptr = ArrayObj::extra_items_mut(container).as_mut_ptr();
+ container.data = base_ptr as *mut _;
+
+ for (i, item) in items.into_iter().enumerate() {
+ write_cell(base_ptr.add(i), item);
+ }
+ }
+ // SAFETY: `arc` was allocated and initialized above as an `Array<T>`.
+ unsafe { Self::from_data(arc) }
+ }
+
+ /// Make this handle the only reference to its cells, with room for at
least
+ /// `capacity` of them, and return the container to edit in place.
+ fn make_mut(&mut self, capacity: usize) -> &mut ArrayObj {
+ // As C++ `Array::CopyOnWrite(reserve)`.
+ let unique = ObjectArc::is_unique(&self.data);
+ let current = self.data.capacity as usize;
+ if !unique || capacity > current {
+ let capacity = if capacity > current {
+ (current * 2).max(capacity)
+ } else {
+ current
+ };
+ self.copy_to_new_buffer(capacity, unique);
+ }
+ // SAFETY: this handle is now the only reference, so no one else
observes
+ // the container while it is mutably borrowed.
+ unsafe { &mut *ObjectArc::as_mut_ptr(&mut self.data) }
+ }
+
+ /// Replace `self` with a unique array of `capacity` cells holding the same
+ /// elements: moved out of a `unique` source, otherwise each re-referenced.
+ fn copy_to_new_buffer(&mut self, capacity: usize, unique: bool) {
+ let size = self.len();
+ let mut grown = Self::new_with_capacity(Vec::new(), capacity);
+ let source = self.as_container().data as *const TVMFFIAny;
+ // SAFETY: `grown` is fresh with room for `size` cells. A unique source
+ // forgets the cells it moved; a shared one keeps its own references.
+ unsafe {
+ let target = ObjectArc::as_mut_ptr(&mut grown.data);
+ let cells = (*target).data as *mut TVMFFIAny;
+ if unique {
+ std::ptr::copy_nonoverlapping(source, cells, size);
+ (*ObjectArc::as_mut_ptr(&mut self.data)).size = 0;
+ } else {
+ for i in 0..size {
+ let copy =
Any::from(AnyView::from_raw_ffi_any(*source.add(i)));
+ std::ptr::write(cells.add(i), Any::into_raw_ffi_any(copy));
+ }
+ }
+ (*target).size = size as i64;
+ }
+ *self = grown;
+ }
+
/// The cells of the array, one per element.
#[inline]
fn cells(&self) -> &[TVMFFIAny] {
@@ -463,6 +456,10 @@ impl<T: ContainerElement + Clone> Array<T> {
}
}
+//
-----------------------------------------------------------------------------
+// Index
+//
-----------------------------------------------------------------------------
+
/// Lends `&T` from the array's cell in place. Every cell of an `Array<T>`
/// holds a `T` (checked when the array is built), so only a debug check runs.
impl<T: ContainerElement + Clone + RefFromAnyUnchecked> std::ops::Index<usize>
for Array<T> {
@@ -485,6 +482,10 @@ impl<T: ContainerElement + Clone + RefFromAnyUnchecked>
std::ops::Index<usize> f
}
}
+//
-----------------------------------------------------------------------------
+// Iterators
+//
-----------------------------------------------------------------------------
+
impl<T: ContainerElement + Clone + RefFromAnyUnchecked> Array<T> {
/// Return an iterator that borrows each element in place, without a
/// reference-count bump, as `array[i]` does.
@@ -602,6 +603,10 @@ impl<T: ContainerElement + Clone> FromIterator<T> for
Array<T> {
}
}
+//
-----------------------------------------------------------------------------
+// Any conversions
+//
-----------------------------------------------------------------------------
+
unsafe impl<T> AnyCompatible for Array<T>
where
T: ContainerElement + Clone,
@@ -770,3 +775,14 @@ where
Ok(TryFromTemp::into_value(temp))
}
}
+
+//
-----------------------------------------------------------------------------
+// Helpers
+//
-----------------------------------------------------------------------------
+
+/// Move `value` into the uninitialized `cell`.
+unsafe fn write_cell<T: ContainerElement>(cell: *mut TVMFFIAny, value: T) {
+ let mut raw = TVMFFIAny::new();
+ T::container_move_to_any(value, &mut raw);
+ std::ptr::write(cell, raw);
+}
diff --git a/rust/tvm-ffi/src/error.rs b/rust/tvm-ffi/src/error.rs
index f4777f06..7df6c6ab 100644
--- a/rust/tvm-ffi/src/error.rs
+++ b/rust/tvm-ffi/src/error.rs
@@ -26,53 +26,6 @@ use tvm_ffi_sys::{
TVMFFIErrorMoveFromRaised, TVMFFIErrorSetRaised, TVMFFIObjectHandle,
TVMFFITypeIndex,
};
-/// The kind of an [`Error`], such as `ValueError`, matching the exception
-/// type that Python raises.
-///
-/// Use the constants such as [`VALUE_ERROR`] and [`TYPE_ERROR`], or
-/// [`ErrorKind::new("MyError")`](ErrorKind::new) for a custom kind.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ErrorKind<'a>(&'a str);
-
-impl<'a> ErrorKind<'a> {
- /// The kind named `name`, which Python raises as that exception type.
- ///
- /// ```
- /// use tvm_ffi::{Error, ErrorKind};
- /// const MY_ERROR: ErrorKind = ErrorKind::new("MyError");
- /// let error = Error::new(MY_ERROR, "bad input", "");
- /// assert_eq!(error.kind().as_str(), "MyError");
- /// ```
- pub const fn new(name: &'a str) -> Self {
- Self(name)
- }
-
- /// Get the name of the kind.
- ///
- /// # Returns
- ///
- /// The name, such as `"ValueError"`.
- pub fn as_str(&self) -> &str {
- self.0
- }
-}
-
-impl<'a> std::fmt::Display for ErrorKind<'a> {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- write!(f, "{}", self.0)
- }
-}
-
-pub const VALUE_ERROR: ErrorKind = ErrorKind::new("ValueError");
-pub const TYPE_ERROR: ErrorKind = ErrorKind::new("TypeError");
-pub const RUNTIME_ERROR: ErrorKind = ErrorKind::new("RuntimeError");
-pub const ATTRIBUTE_ERROR: ErrorKind = ErrorKind::new("AttributeError");
-pub const KEY_ERROR: ErrorKind = ErrorKind::new("KeyError");
-pub const INDEX_ERROR: ErrorKind = ErrorKind::new("IndexError");
-pub const INTERNAL_ERROR: ErrorKind = ErrorKind::new("InternalError");
-pub const OVERFLOW_ERROR: ErrorKind = ErrorKind::new("OverflowError");
-pub const ZERO_DIVISION_ERROR: ErrorKind = ErrorKind::new("ZeroDivisionError");
-
/// The object behind an [`Error`]: its kind, message, backtrace, cause and
context.
#[repr(C)]
#[derive(Object)]
@@ -123,6 +76,53 @@ unsafe impl Sync for Error {}
/// A `Result` whose error type defaults to [`Error`].
pub type Result<T, E = Error> = std::result::Result<T, E>;
+/// The kind of an [`Error`], such as `ValueError`, matching the exception
+/// type that Python raises.
+///
+/// Use the constants such as [`VALUE_ERROR`] and [`TYPE_ERROR`], or
+/// [`ErrorKind::new("MyError")`](ErrorKind::new) for a custom kind.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ErrorKind<'a>(&'a str);
+
+impl<'a> ErrorKind<'a> {
+ /// The kind named `name`, which Python raises as that exception type.
+ ///
+ /// ```
+ /// use tvm_ffi::{Error, ErrorKind};
+ /// const MY_ERROR: ErrorKind = ErrorKind::new("MyError");
+ /// let error = Error::new(MY_ERROR, "bad input", "");
+ /// assert_eq!(error.kind().as_str(), "MyError");
+ /// ```
+ pub const fn new(name: &'a str) -> Self {
+ Self(name)
+ }
+
+ /// Get the name of the kind.
+ ///
+ /// # Returns
+ ///
+ /// The name, such as `"ValueError"`.
+ pub fn as_str(&self) -> &str {
+ self.0
+ }
+}
+
+impl<'a> std::fmt::Display for ErrorKind<'a> {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "{}", self.0)
+ }
+}
+
+pub const VALUE_ERROR: ErrorKind = ErrorKind::new("ValueError");
+pub const TYPE_ERROR: ErrorKind = ErrorKind::new("TypeError");
+pub const RUNTIME_ERROR: ErrorKind = ErrorKind::new("RuntimeError");
+pub const ATTRIBUTE_ERROR: ErrorKind = ErrorKind::new("AttributeError");
+pub const KEY_ERROR: ErrorKind = ErrorKind::new("KeyError");
+pub const INDEX_ERROR: ErrorKind = ErrorKind::new("IndexError");
+pub const INTERNAL_ERROR: ErrorKind = ErrorKind::new("InternalError");
+pub const OVERFLOW_ERROR: ErrorKind = ErrorKind::new("OverflowError");
+pub const ZERO_DIVISION_ERROR: ErrorKind = ErrorKind::new("ZeroDivisionError");
+
impl Error {
/// The `InternalError` for a panic caught at an FFI boundary, where
unwinding
/// into the caller would abort the process. The message is the panic's,
when it
@@ -325,21 +325,6 @@ impl Error {
}
}
- /// Make this handle the only reference to its error, copying a shared
one, as
- /// `Arc::make_mut`.
- fn make_mut(&mut self) -> *mut ErrorObj {
- if !ObjectArc::is_unique(&self.data) {
- *self = Self::new_with_cause_and_extra_context(
- self.kind(),
- self.message(),
- self.backtrace(),
- self.cause_chain(),
- self.extra_context(),
- );
- }
- ObjectArc::as_mut_ptr(&mut self.data)
- }
-
/// Get the traceback of the error in the order of most recent call last,
as
/// Python prints it.
///
@@ -384,6 +369,21 @@ impl Error {
}
this
}
+
+ /// Make this handle the only reference to its error, copying a shared
one, as
+ /// `Arc::make_mut`.
+ fn make_mut(&mut self) -> *mut ErrorObj {
+ if !ObjectArc::is_unique(&self.data) {
+ *self = Self::new_with_cause_and_extra_context(
+ self.kind(),
+ self.message(),
+ self.backtrace(),
+ self.cause_chain(),
+ self.extra_context(),
+ );
+ }
+ ObjectArc::as_mut_ptr(&mut self.data)
+ }
}
impl std::fmt::Display for Error {
diff --git a/rust/tvm-ffi/src/extra/structural_mutate.rs
b/rust/tvm-ffi/src/extra/structural_mutate.rs
index eb348284..a92cd04b 100644
--- a/rust/tvm-ffi/src/extra/structural_mutate.rs
+++ b/rust/tvm-ffi/src/extra/structural_mutate.rs
@@ -72,10 +72,13 @@ const SHALLOW_COPY_ATTR: &str = "__ffi_shallow_copy__";
//
-----------------------------------------------------------------------------
pub(super) type StructuralMutatorHandle = *mut StructuralMutatorObj;
+
type FStructuralMutate =
unsafe extern "C" fn(StructuralMutatorHandle, AnyView<'static>) ->
TVMFFIAny;
+
type FStructuralVarRemapGet =
unsafe extern "C" fn(StructuralMutatorHandle, AnyView<'static>) ->
TVMFFIAny;
+
type FStructuralVarRemapSet =
unsafe extern "C" fn(StructuralMutatorHandle, AnyView<'static>,
AnyView<'static>) -> TVMFFIAny;
@@ -95,6 +98,22 @@ pub struct StructuralMutatorObj {
def_region_mode: UnsafeCell<i32>,
}
+// Scoped runtime prefixes are not independently allocatable ObjectCore values.
+impl StructuralMutatorObj {
+ const TYPE_KEY: &'static str = "ffi.StructuralMutator";
+}
+
+impl Default for StructuralMutatorObj {
+ #[inline(always)]
+ fn default() -> Self {
+ Self {
+ base: Object::new(),
+ vtable: std::ptr::null(),
+ def_region_mode: UnsafeCell::new(DefRegionKind::None as i32),
+ }
+ }
+}
+
const _: () = {
assert!(
std::mem::offset_of!(StructuralMutatorObj, vtable) ==
std::mem::size_of::<TVMFFIObject>()
@@ -105,11 +124,6 @@ const _: () = {
);
};
-// Scoped runtime prefixes are not independently allocatable ObjectCore values.
-impl StructuralMutatorObj {
- const TYPE_KEY: &'static str = "ffi.StructuralMutator";
-}
-
/// Native state and default mutation for [`StructuralMutateEngine`].
/// The engine owns allocation, scoped dispatch, identity remaps and results.
///
@@ -183,17 +197,6 @@ pub struct StructuralMutateEngine<Base =
StructuralMutatorObj> {
remap: RefCell<StructuralVarRemap>,
}
-impl Default for StructuralMutatorObj {
- #[inline(always)]
- fn default() -> Self {
- Self {
- base: Object::new(),
- vtable: std::ptr::null(),
- def_region_mode: UnsafeCell::new(DefRegionKind::None as i32),
- }
- }
-}
-
/// Identity substitutions for structural mutation within one invocation.
///
/// Internal to the engine: handlers reach the remap through
@@ -251,52 +254,6 @@ impl Drop for StructuralVarRemap {
}
}
-impl<'source, 'value, T: crate::type_traits::ContainerElement>
-
super::dispatch::DispatchRecovery<crate::match_any_internal::MutationSource<'source>>
- for crate::MutationView<'value, T>
-{
- type Recovery = crate::MutationView<'source>;
-}
-
-//
-----------------------------------------------------------------------------
-// Public mutation API
-//
-----------------------------------------------------------------------------
-
-mod callback_result_sealed {
- use super::{Any, Result};
-
- pub trait Sealed {}
-
- impl<T: Into<Any>> Sealed for T {}
- impl<T: Into<Any>> Sealed for Result<T> {}
-}
-
-/// Convert an infallible or fallible map or mutation callback result into
-/// [`Result<UnchangedOr<Any>>`].
-///
-/// A callback may return any value convertible into [`Any`], including
-/// [`Unchanged`] and [`UnchangedOr`], or wrap it in [`Result`] to use `?`.
-///
-/// This trait is sealed and is not an extension point.
-#[doc(hidden)]
-pub trait IntoMutateResult: callback_result_sealed::Sealed {
- fn into_mutate_result(self) -> Result<UnchangedOr<Any>>;
-}
-
-impl<T: Into<Any>> IntoMutateResult for T {
- #[inline(always)]
- fn into_mutate_result(self) -> Result<UnchangedOr<Any>> {
- Ok(UnchangedOr::from_any(self.into()))
- }
-}
-
-impl<T: Into<Any>> IntoMutateResult for Result<T> {
- #[inline(always)]
- fn into_mutate_result(self) -> Result<UnchangedOr<Any>> {
- self.map(|value| UnchangedOr::from_any(value.into()))
- }
-}
-
/// Whether a [`MutationView`] may reuse its value's storage in place.
///
/// `Allow` is an upper bound: a value is reused in place only when it is also
@@ -516,7 +473,9 @@ mod mutation_field {
/// Sealed to `Any` and owning object handles; borrowed conversion adapters do
not qualify.
#[doc(hidden)]
pub trait MutationField: crate::type_traits::ContainerElement +
mutation_field::Sealed {}
+
impl MutationField for Any {}
+
impl<T: object::ObjectRefCore> MutationField for T {}
impl<T: object::ObjectRefCore> std::ops::Deref for MutationView<'_, T> {
@@ -601,6 +560,13 @@ impl<T: object::ObjectRefCore> MutationView<'_, T> {
}
}
+impl<'source, 'value, T: crate::type_traits::ContainerElement>
+
super::dispatch::DispatchRecovery<crate::match_any_internal::MutationSource<'source>>
+ for crate::MutationView<'value, T>
+{
+ type Recovery = crate::MutationView<'source>;
+}
+
/// The running structural mutation, passed to each handler.
///
/// A handler that matches a node decides how its children are rewritten: call
@@ -616,6 +582,10 @@ pub struct StructuralMutatorView<'a, D, Base =
StructuralMutatorObj> {
_marker: PhantomData<(&'a mut (), fn(&mut D) -> Base, Rc<()>)>,
}
+//
-----------------------------------------------------------------------------
+// Public mutation API and entrypoint
+//
-----------------------------------------------------------------------------
+
impl<D, Base: StructuralMutatorBase> StructuralMutatorView<'_, D, Base> {
/// Inspect the base state without retaining the scoped mutator runtime.
///
@@ -811,6 +781,120 @@ impl<D, Base: StructuralMutatorBase>
StructuralMutatorView<'_, D, Base> {
}
}
+/// Mutate a structured value with an owned or borrowed [`Dispatcher`].
+///
+/// The root is consumed to establish the ownership boundary for optional
+/// in-place mutation. A matching callback supplies the final value; unmatched
+/// values use default mutation.
+///
+/// # Arguments
+///
+/// * `root` - Owned value to mutate, convertible into an [`Any`]. A uniquely
+/// owned container may be reused in place; clone first to preserve the
original.
+/// * `dispatcher` - Owned or mutably borrowed dispatcher implementing
[`IntoDispatcher`].
+/// Matching handlers return replacements and control recursion through
+/// [`StructuralMutatorView`]; unmatched values use default mutation. For
closure
+/// callbacks with automatic recursion, use [`structural_map`].
+///
+/// # Examples
+///
+/// Double the integers in nested arrays; default mutation descends into the
+/// arrays:
+///
+/// ```
+/// use tvm_ffi::{
+/// dispatch, structural_mutate, Any, Array, MutationView, Result,
StructuralMutatorView,
+/// UnchangedOr,
+/// };
+///
+/// struct Double;
+///
+/// #[dispatch(mutate, (MutationView<'_>, &mut StructuralMutatorView<'_,
Self>), Result<UnchangedOr<Any>>)]
+/// impl Double {
+/// fn mutate_int(
+/// &mut self,
+/// value: i64,
+/// _: &mut StructuralMutatorView<'_, Self>,
+/// ) -> Result<UnchangedOr<Any>> {
+/// Ok((value * 2).into())
+/// }
+/// }
+///
+/// let root = Array::from_vec(vec![Array::from_vec(vec![1_i64, 2]),
Array::from_vec(vec![3_i64])]);
+/// let result = Array::<Array<i64>>::try_from(structural_mutate(root,
Double)?)?;
+/// assert_eq!(result.get(0)?.iter().copied().collect::<Vec<_>>(), vec![2, 4]);
+/// # Ok::<(), tvm_ffi::Error>(())
+/// ```
+pub fn structural_mutate<R, H, M, D, O>(root: R, dispatcher: H) -> Result<Any>
+where
+ R: Into<Any>,
+ O: BorrowMut<D>,
+ for<'v, 'c, 'r> H: IntoDispatcher<
+ (
+ MutationView<'v>,
+ &'c mut StructuralMutatorView<'r, D, StructuralMutatorObj>,
+ ),
+ Option<Result<UnchangedOr<Any>>>,
+ M,
+ Target = D,
+ Output = O,
+ >,
+ for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
+ (
+ MutationView<'v>,
+ &'c mut StructuralMutatorView<'r, D, StructuralMutatorObj>,
+ ),
+ Option<Result<UnchangedOr<Any>>>,
+ >,
+{
+ StructuralMutateEngine::mutate_root(root, dispatcher)
+}
+
+mod callback_result_sealed {
+ use super::{Any, Result};
+
+ pub trait Sealed {}
+
+ impl<T: Into<Any>> Sealed for T {}
+ impl<T: Into<Any>> Sealed for Result<T> {}
+}
+
+/// Convert an infallible or fallible map or mutation callback result into
+/// [`Result<UnchangedOr<Any>>`].
+///
+/// A callback may return any value convertible into [`Any`], including
+/// [`Unchanged`] and [`UnchangedOr`], or wrap it in [`Result`] to use `?`.
+///
+/// This trait is sealed and is not an extension point.
+#[doc(hidden)]
+pub trait IntoMutateResult: callback_result_sealed::Sealed {
+ fn into_mutate_result(self) -> Result<UnchangedOr<Any>>;
+}
+
+impl<T: Into<Any>> IntoMutateResult for T {
+ #[inline(always)]
+ fn into_mutate_result(self) -> Result<UnchangedOr<Any>> {
+ Ok(UnchangedOr::from_any(self.into()))
+ }
+}
+
+impl<T: Into<Any>> IntoMutateResult for Result<T> {
+ #[inline(always)]
+ fn into_mutate_result(self) -> Result<UnchangedOr<Any>> {
+ self.map(|value| UnchangedOr::from_any(value.into()))
+ }
+}
+
+//
-----------------------------------------------------------------------------
+// StructuralMutateEngine implementation
+//
-----------------------------------------------------------------------------
+
+// Perf note: Raw carriers avoid materializing an owning Any on the success
path.
+// Private C-unwind boundaries return this 16-byte carrier in x0/x1 on AArch64,
+// avoiding the hidden result slot observed with the Rust ABI while allowing
+// Rust cleanup to unwind. Native vtables remain extern "C".
+// Raw bits provide no ownership guard: import an owning result into Any before
+// fallible work or destruction of an observer; borrowed inputs must stay live.
impl<Base: StructuralMutatorBase> StructuralMutateEngine<Base> {
/// Mutate `root` with `dispatcher` on the native base `Base`. See also
[`structural_mutate`].
#[inline(always)]
@@ -835,317 +919,35 @@ impl<Base: StructuralMutatorBase>
StructuralMutateEngine<Base> {
let mut storage = dispatcher.into_dispatcher();
Self::run_mutator::<D, true>(root.into(), storage.borrow_mut())
}
-}
-//
-----------------------------------------------------------------------------
-// Outlined errors
-//
-----------------------------------------------------------------------------
+ /// The engine whose first field is the native prefix `runtime`.
+ ///
+ /// # Safety
+ ///
+ /// `runtime` is this live Rust engine, not an arbitrary native mutator;
only raw
+ /// projections of the result may cross recursion.
+ #[inline(always)]
+ unsafe fn from_handle(runtime: StructuralMutatorHandle) -> *mut Self {
+ runtime.cast()
+ }
-#[cold]
-#[inline(never)]
-fn missing_shallow_copy_error(type_index: i32, attribute: &str) -> Error {
- Error::new(
- TYPE_ERROR,
- &format!(
- "type `{}` cannot use reflected structural mutation because it
does not define `{attribute}`",
- crate::object::type_key_of_index(type_index)
- ),
- "",
- )
-}
-
-#[cold]
-#[inline(never)]
-fn invalid_shallow_copy_hook_error(attribute: &str) -> Error {
- Error::new(
- TYPE_ERROR,
- &format!("{attribute} must be an ffi.Function"),
- "",
- )
-}
-
-#[cold]
-#[inline(never)]
-fn invalid_shallow_copy_result_error() -> Error {
- Error::new(
- TYPE_ERROR,
- "shallow copy callback must return a distinct object with the same
type as its input",
- "",
- )
-}
-
-#[cold]
-#[inline(never)]
-fn missing_field_setter_error(name: &str) -> Error {
- Error::new(
- TYPE_ERROR,
- &format!("cannot structurally mutate field `{name}` because it does
not define a setter"),
- "",
- )
-}
-
-#[cold]
-#[inline(never)]
-fn invalid_field_offset_error(name: &str) -> Error {
- runtime_error(&format!(
- "native structural mutate: reflected field `{name}` has an invalid
offset"
- ))
-}
-
-#[cold]
-#[inline(never)]
-fn invalid_mutation_hook_error() -> Error {
- Error::new(
- TYPE_ERROR,
- "__s_mutate__ must be an opaque function pointer or ffi.Function",
- "",
- )
-}
-
-#[cold]
-#[inline(never)]
-fn var_remap_key_error() -> Error {
- Error::new(
- TYPE_ERROR,
- "variable-remap keys must be object-backed values",
- "",
- )
-}
-
-#[cold]
-#[inline(never)]
-fn unregistered_type_error(type_index: i32) -> Error {
- runtime_error(&format!(
- "native structural mutate: unregistered type index {type_index}"
- ))
-}
-
-#[cold]
-pub(super) fn with_mutation_error_context(error: Error, raw: TVMFFIAny,
operation: &str) -> Error {
- if raw.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
- error
- } else {
- with_structural_error_context(
- update_visit_error_context(error, raw),
- operation,
- &format!(
- "object `{}`",
- crate::object::type_key_of_index(raw.type_index)
- ),
- )
- }
-}
-
-//
-----------------------------------------------------------------------------
-// Private utility functions
-//
-----------------------------------------------------------------------------
-
-fn shallow_copy(raw: TVMFFIAny) -> Result<Any> {
- let Some(attr) = SHALLOW_COPY_COLUMN.get_raw_ffi_any(raw.type_index) else {
- return Err(missing_shallow_copy_error(
- raw.type_index,
- SHALLOW_COPY_ATTR,
- ));
- };
- if attr.type_index != TVMFFITypeIndex::kTVMFFIFunction as i32 {
- return Err(invalid_shallow_copy_hook_error(SHALLOW_COPY_ATTR));
- }
- let function = unsafe { attr.data_union.v_obj };
- if function.is_null() {
- return Err(runtime_error("shallow copy function pointer is null"));
- }
- // The registry owns this function throughout the call; borrowing it avoids
- // an atomic retain/release for every reflected node.
- let mut result = Any::new();
- let status = unsafe {
- TVMFFIFunctionCall(
- function.cast(),
- &raw,
- 1,
- Any::as_raw_ffi_any_mut(&mut result),
- )
- };
- if status != 0 {
- return Err(Error::from_raised());
- }
- let result_raw = *result.as_raw_ffi_any();
- let result_pointer = unsafe { result_raw.data_union.v_obj };
- let source_pointer = unsafe { raw.data_union.v_obj };
- if result_raw.type_index != raw.type_index
- || result_pointer.is_null()
- || result_pointer == source_pointer
- {
- return Err(invalid_shallow_copy_result_error());
- }
- Ok(result)
-}
-
-fn call_field_setter(
- field: &TVMFFIFieldInfo,
- field_address: *mut c_void,
- value: &TVMFFIAny,
-) -> Result<()> {
- if field.setter.is_null() {
- return Err(missing_field_setter_error(field.name.as_str()));
- }
- let return_code = unsafe {
- if field.flags & FLAG_SETTER_IS_FUNCTION == 0 {
- // SAFETY: reflection registration requires a non-Function setter
- // pointer to use the TVMFFIFieldSetter signature.
- let setter: TVMFFIFieldSetter = std::mem::transmute(field.setter);
- setter(field_address, value)
- } else {
- let mut args = [TVMFFIAny::new(), *value];
- args[0].type_index = TVMFFITypeIndex::kTVMFFIOpaquePtr as i32;
- args[0].data_union.v_ptr = field_address;
- // Own the result slot before entering foreign code so a partial
- // owning result is released on both success and failure.
- let mut result = Any::new();
- TVMFFIFunctionCall(
- field.setter as TVMFFIObjectHandle,
- args.as_ptr(),
- 2,
- Any::as_raw_ffi_any_mut(&mut result),
- )
- }
- };
- if return_code == 0 {
- Ok(())
- } else {
- Err(Error::from_raised())
- }
-}
-
-fn object_identity_key(raw: TVMFFIAny) -> Result<NonNull<TVMFFIObject>> {
- if raw.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
- core::hint::cold_path();
- return Err(var_remap_key_error());
- }
- let pointer = unsafe { raw.data_union.v_obj };
- NonNull::new(pointer).ok_or_else(|| {
- core::hint::cold_path();
- runtime_error("native structural mutate: identity object has a null
pointer")
- })
-}
-
-#[inline]
-fn checked_type_info(type_index: i32) -> Result<*const
crate::tvm_ffi_sys::TVMFFITypeInfo> {
- let info = unsafe { TVMFFIGetTypeInfo(type_index) };
- if info.is_null() {
- Err(unregistered_type_error(type_index))
- } else {
- Ok(info)
- }
-}
-
-#[inline(always)]
-fn object_is_unique(raw: TVMFFIAny) -> bool {
- if raw.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
- return false;
- }
- let pointer = unsafe { raw.data_union.v_obj };
- !pointer.is_null() && unsafe { object::unsafe_::is_unique(pointer) }
-}
-
-/// Acquire an owning value from a valid borrowed carrier.
-///
-/// Inline values are copied and objects are retained once, including Error
objects
-/// as ordinary values. String/byte views are normalized by the runtime; an
rvalue
-/// slot transfers its reference. Internal callers keep borrowed resources
live and
-/// uniquely own each writable rvalue slot, converting it at most once. A null
-/// object pointer or failed runtime conversion returns an error; successful
Error
-/// objects remain ordinary values rather than entering the error channel.
-fn raw_ffi_any_to_owned_any(raw: TVMFFIAny) -> Result<Any> {
- if is_plain_inline(raw.type_index) {
- return Ok(unsafe { Any::from_raw_ffi_any(raw) });
- }
- if raw.type_index >= TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
- let object = unsafe { raw.data_union.v_obj };
- if object.is_null() {
- return Err(runtime_error(
- "native structural mutate: object-backed value has a null
pointer",
- ));
- }
- unsafe { object::unsafe_::inc_ref(object) };
- return Ok(unsafe { Any::from_raw_ffi_any(raw) });
- }
-
- // Raw string/bytes views and ObjectRValueRef require normalization (or a
- // move) rather than a bitwise copy; keep the generic C ABI conversion for
- // those uncommon representations.
- let mut owned = Any::new();
- let return_code = unsafe { TVMFFIAnyViewToOwnedAny(&raw,
Any::as_raw_ffi_any_mut(&mut owned)) };
- if return_code == 0 {
- Ok(owned)
- } else {
- Err(Error::from_raised())
- }
-}
-
-// Cache registry columns, not their optional per-type cells. Absent cells
retain
-// the normal hook/reflection fallback.
-static STRUCTURAL_MUTATE_COLUMN: ColdInitLazyLock<TypeAttrColumn> =
ColdInitLazyLock::new(|| {
- TypeAttrColumn::new(STRUCTURAL_MUTATE_ATTR).expect("missing __s_mutate__
column")
-});
-static STRUCTURAL_MAYBE_INPLACE_MUTATE_COLUMN:
ColdInitLazyLock<TypeAttrColumn> =
- ColdInitLazyLock::new(|| {
- TypeAttrColumn::new(STRUCTURAL_MAYBE_INPLACE_MUTATE_ATTR)
- .expect("missing __s_maybe_inplace_mutate__ column")
- });
-/// Return the shared column for normal or permitted in-place mutation hooks.
-/// Cache the column, not its per-type cells, so absent hooks still use
fallback.
-#[inline(always)]
-fn structural_mutate_column<const INPLACE: bool>() -> TypeAttrColumn {
- if INPLACE {
- *STRUCTURAL_MAYBE_INPLACE_MUTATE_COLUMN
- } else {
- *STRUCTURAL_MUTATE_COLUMN
- }
-}
-
-static SHALLOW_COPY_COLUMN: ColdInitLazyLock<TypeAttrColumn> =
ColdInitLazyLock::new(|| {
- TypeAttrColumn::new(SHALLOW_COPY_ATTR).expect("missing
__ffi_shallow_copy__ column")
-});
-
-//
-----------------------------------------------------------------------------
-// StructuralMutateEngine implementation
-//
-----------------------------------------------------------------------------
-
-// Perf note: Raw carriers avoid materializing an owning Any on the success
path.
-// Private C-unwind boundaries return this 16-byte carrier in x0/x1 on AArch64,
-// avoiding the hidden result slot observed with the Rust ABI while allowing
-// Rust cleanup to unwind. Native vtables remain extern "C".
-// Raw bits provide no ownership guard: import an owning result into Any before
-// fallible work or destruction of an observer; borrowed inputs must stay live.
-impl<Base: StructuralMutatorBase> StructuralMutateEngine<Base> {
- /// The engine whose first field is the native prefix `runtime`.
- ///
- /// # Safety
- ///
- /// `runtime` is this live Rust engine, not an arbitrary native mutator;
only raw
- /// projections of the result may cross recursion.
- #[inline(always)]
- unsafe fn from_handle(runtime: StructuralMutatorHandle) -> *mut Self {
- runtime.cast()
- }
-
- /// Validate a recursive caller and return its current exclusive reborrow.
- ///
- /// # Safety
- /// `runtime` must belong to the live invocation of `dispatch` with
matching `D`.
- /// The identity check does not replace that lifetime/type contract;
mismatch panics.
- #[inline(always)]
- unsafe fn checked_dispatcher<D>(
- runtime: StructuralMutatorHandle,
- dispatch: &mut D,
- ) -> *mut c_void {
- let context = std::ptr::from_mut(dispatch).cast();
- if context != (*Self::from_handle(runtime)).dispatcher {
- core::hint::cold_path();
- wrong_dispatcher();
- }
- context
- }
+ /// Validate a recursive caller and return its current exclusive reborrow.
+ ///
+ /// # Safety
+ /// `runtime` must belong to the live invocation of `dispatch` with
matching `D`.
+ /// The identity check does not replace that lifetime/type contract;
mismatch panics.
+ #[inline(always)]
+ unsafe fn checked_dispatcher<D>(
+ runtime: StructuralMutatorHandle,
+ dispatch: &mut D,
+ ) -> *mut c_void {
+ let context = std::ptr::from_mut(dispatch).cast();
+ if context != (*Self::from_handle(runtime)).dispatcher {
+ core::hint::cold_path();
+ wrong_dispatcher();
+ }
+ context
+ }
/// Build native callbacks for dispatcher `D`. `CALLBACK_CONTEXT` selects
/// whether direct callback failures receive input context; map adapters
add their own.
@@ -1349,6 +1151,10 @@ impl<Base: StructuralMutatorBase>
StructuralMutateEngine<Base> {
}
}
+//
-----------------------------------------------------------------------------
+// Prefix-only default traversal and native ABI operations
+//
-----------------------------------------------------------------------------
+
// Prefix operations use only the native layout and installed vtable.
impl StructuralMutatorObj {
/// Consume a native hook's `result`, naming the borrowed `raw` in it when
it is an Error.
@@ -1670,131 +1476,330 @@ impl StructuralMutatorObj {
Ok((value.type_index() != TVMFFITypeIndex::kTVMFFINone as
i32).then_some(value))
}
- /// Borrow both inputs through the native remap interface; it retains the
key
- /// and replacement. Import the owning None/Error result before returning.
- #[inline(always)]
- unsafe fn var_remap_set(
- runtime: StructuralMutatorHandle,
- raw: TVMFFIAny,
- value: &Any,
- ) -> Result<()> {
- let callback = (*(*runtime).vtable).var_remap_set;
- raw_ffi_any_into_result(callback(
- runtime,
- AnyView::from_raw_ffi_any(raw),
- AnyView::from_raw_ffi_any(*value.as_raw_ffi_any()),
- ))
- .map(|_| ())
+ /// Borrow both inputs through the native remap interface; it retains the
key
+ /// and replacement. Import the owning None/Error result before returning.
+ #[inline(always)]
+ unsafe fn var_remap_set(
+ runtime: StructuralMutatorHandle,
+ raw: TVMFFIAny,
+ value: &Any,
+ ) -> Result<()> {
+ let callback = (*(*runtime).vtable).var_remap_set;
+ raw_ffi_any_into_result(callback(
+ runtime,
+ AnyView::from_raw_ffi_any(raw),
+ AnyView::from_raw_ffi_any(*value.as_raw_ffi_any()),
+ ))
+ .map(|_| ())
+ }
+
+ /// The reflective mutation of [`Self::mutate_without_hook_raw`], on owned
results.
+ #[inline(always)]
+ fn mutate_without_hook(
+ runtime: StructuralMutatorHandle,
+ raw: TVMFFIAny,
+ def_region_kind: DefRegionKind,
+ ) -> Result<Any> {
+ if raw.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
+ return raw_ffi_any_to_owned_any(raw);
+ }
+
+ let type_info = checked_type_info(raw.type_index)?;
+ let kind = unsafe { seq_hash_kind(type_info) };
+ let is_free_var = kind == TVMFFISEqHashKind::kTVMFFISEqHashKindFreeVar
as i32;
+ let is_dag_node = kind == TVMFFISEqHashKind::kTVMFFISEqHashKindDAGNode
as i32;
+ if is_free_var || is_dag_node {
+ // The ABI uses None for a miss; an Unchanged marker is a cached
result.
+ if let Some(mutated) = unsafe { Self::var_remap_get(runtime, raw)?
} {
+ return Ok(mutated);
+ }
+ }
+ // A variable use with no binding retains its identity without
visiting fields.
+ if is_free_var && def_region_kind == DefRegionKind::None {
+ return Ok(Unchanged.into());
+ }
+
+ let result = Self::map_reflected(runtime, raw, def_region_kind,
type_info)?;
+ if is_dag_node
+ || (is_free_var
+ && (def_region_kind == DefRegionKind::Pattern ||
!is_unchanged(&result)))
+ {
+ // Keep markers for DAG nodes and pattern definitions. An
unchanged simple
+ // definition needs no binding: subsequent uses retain the
original on a miss.
+ unsafe { Self::var_remap_set(runtime, raw, &result)? };
+ }
+ Ok(result)
+ }
+}
+
+//
-----------------------------------------------------------------------------
+// Private utility functions
+//
-----------------------------------------------------------------------------
+
+fn shallow_copy(raw: TVMFFIAny) -> Result<Any> {
+ let Some(attr) = SHALLOW_COPY_COLUMN.get_raw_ffi_any(raw.type_index) else {
+ return Err(missing_shallow_copy_error(
+ raw.type_index,
+ SHALLOW_COPY_ATTR,
+ ));
+ };
+ if attr.type_index != TVMFFITypeIndex::kTVMFFIFunction as i32 {
+ return Err(invalid_shallow_copy_hook_error(SHALLOW_COPY_ATTR));
+ }
+ let function = unsafe { attr.data_union.v_obj };
+ if function.is_null() {
+ return Err(runtime_error("shallow copy function pointer is null"));
+ }
+ // The registry owns this function throughout the call; borrowing it avoids
+ // an atomic retain/release for every reflected node.
+ let mut result = Any::new();
+ let status = unsafe {
+ TVMFFIFunctionCall(
+ function.cast(),
+ &raw,
+ 1,
+ Any::as_raw_ffi_any_mut(&mut result),
+ )
+ };
+ if status != 0 {
+ return Err(Error::from_raised());
+ }
+ let result_raw = *result.as_raw_ffi_any();
+ let result_pointer = unsafe { result_raw.data_union.v_obj };
+ let source_pointer = unsafe { raw.data_union.v_obj };
+ if result_raw.type_index != raw.type_index
+ || result_pointer.is_null()
+ || result_pointer == source_pointer
+ {
+ return Err(invalid_shallow_copy_result_error());
+ }
+ Ok(result)
+}
+
+fn call_field_setter(
+ field: &TVMFFIFieldInfo,
+ field_address: *mut c_void,
+ value: &TVMFFIAny,
+) -> Result<()> {
+ if field.setter.is_null() {
+ return Err(missing_field_setter_error(field.name.as_str()));
+ }
+ let return_code = unsafe {
+ if field.flags & FLAG_SETTER_IS_FUNCTION == 0 {
+ // SAFETY: reflection registration requires a non-Function setter
+ // pointer to use the TVMFFIFieldSetter signature.
+ let setter: TVMFFIFieldSetter = std::mem::transmute(field.setter);
+ setter(field_address, value)
+ } else {
+ let mut args = [TVMFFIAny::new(), *value];
+ args[0].type_index = TVMFFITypeIndex::kTVMFFIOpaquePtr as i32;
+ args[0].data_union.v_ptr = field_address;
+ // Own the result slot before entering foreign code so a partial
+ // owning result is released on both success and failure.
+ let mut result = Any::new();
+ TVMFFIFunctionCall(
+ field.setter as TVMFFIObjectHandle,
+ args.as_ptr(),
+ 2,
+ Any::as_raw_ffi_any_mut(&mut result),
+ )
+ }
+ };
+ if return_code == 0 {
+ Ok(())
+ } else {
+ Err(Error::from_raised())
+ }
+}
+
+fn object_identity_key(raw: TVMFFIAny) -> Result<NonNull<TVMFFIObject>> {
+ if raw.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
+ core::hint::cold_path();
+ return Err(var_remap_key_error());
+ }
+ let pointer = unsafe { raw.data_union.v_obj };
+ NonNull::new(pointer).ok_or_else(|| {
+ core::hint::cold_path();
+ runtime_error("native structural mutate: identity object has a null
pointer")
+ })
+}
+
+#[inline]
+fn checked_type_info(type_index: i32) -> Result<*const
crate::tvm_ffi_sys::TVMFFITypeInfo> {
+ let info = unsafe { TVMFFIGetTypeInfo(type_index) };
+ if info.is_null() {
+ Err(unregistered_type_error(type_index))
+ } else {
+ Ok(info)
+ }
+}
+
+#[inline(always)]
+fn object_is_unique(raw: TVMFFIAny) -> bool {
+ if raw.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
+ return false;
+ }
+ let pointer = unsafe { raw.data_union.v_obj };
+ !pointer.is_null() && unsafe { object::unsafe_::is_unique(pointer) }
+}
+
+/// Acquire an owning value from a valid borrowed carrier.
+///
+/// Inline values are copied and objects are retained once, including Error
objects
+/// as ordinary values. String/byte views are normalized by the runtime; an
rvalue
+/// slot transfers its reference. Internal callers keep borrowed resources
live and
+/// uniquely own each writable rvalue slot, converting it at most once. A null
+/// object pointer or failed runtime conversion returns an error; successful
Error
+/// objects remain ordinary values rather than entering the error channel.
+fn raw_ffi_any_to_owned_any(raw: TVMFFIAny) -> Result<Any> {
+ if is_plain_inline(raw.type_index) {
+ return Ok(unsafe { Any::from_raw_ffi_any(raw) });
+ }
+ if raw.type_index >= TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
+ let object = unsafe { raw.data_union.v_obj };
+ if object.is_null() {
+ return Err(runtime_error(
+ "native structural mutate: object-backed value has a null
pointer",
+ ));
+ }
+ unsafe { object::unsafe_::inc_ref(object) };
+ return Ok(unsafe { Any::from_raw_ffi_any(raw) });
+ }
+
+ // Raw string/bytes views and ObjectRValueRef require normalization (or a
+ // move) rather than a bitwise copy; keep the generic C ABI conversion for
+ // those uncommon representations.
+ let mut owned = Any::new();
+ let return_code = unsafe { TVMFFIAnyViewToOwnedAny(&raw,
Any::as_raw_ffi_any_mut(&mut owned)) };
+ if return_code == 0 {
+ Ok(owned)
+ } else {
+ Err(Error::from_raised())
}
+}
- /// The reflective mutation of [`Self::mutate_without_hook_raw`], on owned
results.
- #[inline(always)]
- fn mutate_without_hook(
- runtime: StructuralMutatorHandle,
- raw: TVMFFIAny,
- def_region_kind: DefRegionKind,
- ) -> Result<Any> {
- if raw.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
- return raw_ffi_any_to_owned_any(raw);
- }
+// Cache registry columns, not their optional per-type cells. Absent cells
retain
+// the normal hook/reflection fallback.
+static STRUCTURAL_MUTATE_COLUMN: ColdInitLazyLock<TypeAttrColumn> =
ColdInitLazyLock::new(|| {
+ TypeAttrColumn::new(STRUCTURAL_MUTATE_ATTR).expect("missing __s_mutate__
column")
+});
- let type_info = checked_type_info(raw.type_index)?;
- let kind = unsafe { seq_hash_kind(type_info) };
- let is_free_var = kind == TVMFFISEqHashKind::kTVMFFISEqHashKindFreeVar
as i32;
- let is_dag_node = kind == TVMFFISEqHashKind::kTVMFFISEqHashKindDAGNode
as i32;
- if is_free_var || is_dag_node {
- // The ABI uses None for a miss; an Unchanged marker is a cached
result.
- if let Some(mutated) = unsafe { Self::var_remap_get(runtime, raw)?
} {
- return Ok(mutated);
- }
- }
- // A variable use with no binding retains its identity without
visiting fields.
- if is_free_var && def_region_kind == DefRegionKind::None {
- return Ok(Unchanged.into());
- }
+static STRUCTURAL_MAYBE_INPLACE_MUTATE_COLUMN:
ColdInitLazyLock<TypeAttrColumn> =
+ ColdInitLazyLock::new(|| {
+ TypeAttrColumn::new(STRUCTURAL_MAYBE_INPLACE_MUTATE_ATTR)
+ .expect("missing __s_maybe_inplace_mutate__ column")
+ });
- let result = Self::map_reflected(runtime, raw, def_region_kind,
type_info)?;
- if is_dag_node
- || (is_free_var
- && (def_region_kind == DefRegionKind::Pattern ||
!is_unchanged(&result)))
- {
- // Keep markers for DAG nodes and pattern definitions. An
unchanged simple
- // definition needs no binding: subsequent uses retain the
original on a miss.
- unsafe { Self::var_remap_set(runtime, raw, &result)? };
- }
- Ok(result)
+/// Return the shared column for normal or permitted in-place mutation hooks.
+/// Cache the column, not its per-type cells, so absent hooks still use
fallback.
+#[inline(always)]
+fn structural_mutate_column<const INPLACE: bool>() -> TypeAttrColumn {
+ if INPLACE {
+ *STRUCTURAL_MAYBE_INPLACE_MUTATE_COLUMN
+ } else {
+ *STRUCTURAL_MUTATE_COLUMN
}
}
+static SHALLOW_COPY_COLUMN: ColdInitLazyLock<TypeAttrColumn> =
ColdInitLazyLock::new(|| {
+ TypeAttrColumn::new(SHALLOW_COPY_ATTR).expect("missing
__ffi_shallow_copy__ column")
+});
+
//
-----------------------------------------------------------------------------
-// Public entrypoint
+// Outlined errors
//
-----------------------------------------------------------------------------
-/// Mutate a structured value with an owned or borrowed [`Dispatcher`].
-///
-/// The root is consumed to establish the ownership boundary for optional
-/// in-place mutation. A matching callback supplies the final value; unmatched
-/// values use default mutation.
-///
-/// # Arguments
-///
-/// * `root` - Owned value to mutate, convertible into an [`Any`]. A uniquely
-/// owned container may be reused in place; clone first to preserve the
original.
-/// * `dispatcher` - Owned or mutably borrowed dispatcher implementing
[`IntoDispatcher`].
-/// Matching handlers return replacements and control recursion through
-/// [`StructuralMutatorView`]; unmatched values use default mutation. For
closure
-/// callbacks with automatic recursion, use [`structural_map`].
-///
-/// # Examples
-///
-/// Double the integers in nested arrays; default mutation descends into the
-/// arrays:
-///
-/// ```
-/// use tvm_ffi::{
-/// dispatch, structural_mutate, Any, Array, MutationView, Result,
StructuralMutatorView,
-/// UnchangedOr,
-/// };
-///
-/// struct Double;
-///
-/// #[dispatch(mutate, (MutationView<'_>, &mut StructuralMutatorView<'_,
Self>), Result<UnchangedOr<Any>>)]
-/// impl Double {
-/// fn mutate_int(
-/// &mut self,
-/// value: i64,
-/// _: &mut StructuralMutatorView<'_, Self>,
-/// ) -> Result<UnchangedOr<Any>> {
-/// Ok((value * 2).into())
-/// }
-/// }
-///
-/// let root = Array::from_vec(vec![Array::from_vec(vec![1_i64, 2]),
Array::from_vec(vec![3_i64])]);
-/// let result = Array::<Array<i64>>::try_from(structural_mutate(root,
Double)?)?;
-/// assert_eq!(result.get(0)?.iter().copied().collect::<Vec<_>>(), vec![2, 4]);
-/// # Ok::<(), tvm_ffi::Error>(())
-/// ```
-pub fn structural_mutate<R, H, M, D, O>(root: R, dispatcher: H) -> Result<Any>
-where
- R: Into<Any>,
- O: BorrowMut<D>,
- for<'v, 'c, 'r> H: IntoDispatcher<
- (
- MutationView<'v>,
- &'c mut StructuralMutatorView<'r, D, StructuralMutatorObj>,
- ),
- Option<Result<UnchangedOr<Any>>>,
- M,
- Target = D,
- Output = O,
- >,
- for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
- (
- MutationView<'v>,
- &'c mut StructuralMutatorView<'r, D, StructuralMutatorObj>,
+#[cold]
+#[inline(never)]
+fn missing_shallow_copy_error(type_index: i32, attribute: &str) -> Error {
+ Error::new(
+ TYPE_ERROR,
+ &format!(
+ "type `{}` cannot use reflected structural mutation because it
does not define `{attribute}`",
+ crate::object::type_key_of_index(type_index)
),
- Option<Result<UnchangedOr<Any>>>,
- >,
-{
- StructuralMutateEngine::mutate_root(root, dispatcher)
+ "",
+ )
+}
+
+#[cold]
+#[inline(never)]
+fn invalid_shallow_copy_hook_error(attribute: &str) -> Error {
+ Error::new(
+ TYPE_ERROR,
+ &format!("{attribute} must be an ffi.Function"),
+ "",
+ )
+}
+
+#[cold]
+#[inline(never)]
+fn invalid_shallow_copy_result_error() -> Error {
+ Error::new(
+ TYPE_ERROR,
+ "shallow copy callback must return a distinct object with the same
type as its input",
+ "",
+ )
+}
+
+#[cold]
+#[inline(never)]
+fn missing_field_setter_error(name: &str) -> Error {
+ Error::new(
+ TYPE_ERROR,
+ &format!("cannot structurally mutate field `{name}` because it does
not define a setter"),
+ "",
+ )
+}
+
+#[cold]
+#[inline(never)]
+fn invalid_field_offset_error(name: &str) -> Error {
+ runtime_error(&format!(
+ "native structural mutate: reflected field `{name}` has an invalid
offset"
+ ))
+}
+
+#[cold]
+#[inline(never)]
+fn invalid_mutation_hook_error() -> Error {
+ Error::new(
+ TYPE_ERROR,
+ "__s_mutate__ must be an opaque function pointer or ffi.Function",
+ "",
+ )
+}
+
+#[cold]
+#[inline(never)]
+fn var_remap_key_error() -> Error {
+ Error::new(
+ TYPE_ERROR,
+ "variable-remap keys must be object-backed values",
+ "",
+ )
+}
+
+#[cold]
+#[inline(never)]
+fn unregistered_type_error(type_index: i32) -> Error {
+ runtime_error(&format!(
+ "native structural mutate: unregistered type index {type_index}"
+ ))
+}
+
+#[cold]
+pub(super) fn with_mutation_error_context(error: Error, raw: TVMFFIAny,
operation: &str) -> Error {
+ if raw.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
+ error
+ } else {
+ with_structural_error_context(
+ update_visit_error_context(error, raw),
+ operation,
+ &format!(
+ "object `{}`",
+ crate::object::type_key_of_index(raw.type_index)
+ ),
+ )
+ }
}
diff --git a/rust/tvm-ffi/src/extra/structural_visit.rs
b/rust/tvm-ffi/src/extra/structural_visit.rs
index 1a72734b..2fdfe126 100644
--- a/rust/tvm-ffi/src/extra/structural_visit.rs
+++ b/rust/tvm-ffi/src/extra/structural_visit.rs
@@ -86,6 +86,7 @@ const STRUCTURAL_VISIT_ATTR: &str = "__s_visit__";
//
-----------------------------------------------------------------------------
type StructuralVisitorHandle = *mut StructuralVisitorObj;
+
type FStructuralVisit =
unsafe extern "C" fn(StructuralVisitorHandle, AnyView<'static>) ->
TVMFFIAny;
@@ -103,6 +104,22 @@ pub struct StructuralVisitorObj {
def_region_mode: UnsafeCell<i32>,
}
+// Scoped runtime prefixes are not independently allocatable ObjectCore values.
+impl StructuralVisitorObj {
+ const TYPE_KEY: &'static str = "ffi.StructuralVisitor";
+}
+
+impl Default for StructuralVisitorObj {
+ #[inline(always)]
+ fn default() -> Self {
+ Self {
+ base: Object::new(),
+ vtable: std::ptr::null(),
+ def_region_mode: UnsafeCell::new(DefRegionKind::None as i32),
+ }
+ }
+}
+
/// Native `ffi.VisitInterrupt` object containing the interruption payload.
#[repr(C)]
#[derive(Object)]
@@ -126,9 +143,66 @@ const _: () = {
assert!(std::mem::offset_of!(VisitInterruptObj, value) ==
std::mem::size_of::<TVMFFIObject>());
};
-// Scoped runtime prefixes are not independently allocatable ObjectCore values.
-impl StructuralVisitorObj {
- const TYPE_KEY: &'static str = "ffi.StructuralVisitor";
+/// Definition-region state active at the current value.
+///
+/// Reflected fields marked `SEqHashDefPattern` or
+/// `SEqHashDefSimple` override the inherited state for that field's
+/// complete recursive visit.
+#[repr(i32)]
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub enum DefRegionKind {
+ /// The value is outside a definition region.
+ #[default]
+ None = 0,
+ /// Definitions apply recursively through the visited value.
+ Pattern = 1,
+ /// Definitions apply to the visited value alone; its type is walked as
uses.
+ Simple = 2,
+}
+
+const _: () = {
+ assert!(DefRegionKind::None as i32 ==
TVMFFIDefRegionKind::kTVMFFIDefRegionKindNone as i32);
+ assert!(
+ DefRegionKind::Pattern as i32 ==
TVMFFIDefRegionKind::kTVMFFIDefRegionKindPattern as i32
+ );
+ assert!(DefRegionKind::Simple as i32 ==
TVMFFIDefRegionKind::kTVMFFIDefRegionKindSimple as i32);
+};
+
+/// Interrupt state of a traversal, mirroring C++ `ffi.VisitInterrupt`.
+///
+/// Entry points and visitor-layer calls return
+/// `Result<Option<VisitInterrupt>>`: `Ok(None)` means the (sub)graph was
+/// visited completely, `Ok(Some(..))` means a handler halted the traversal
+/// with this interrupt, and `Err` means it failed.
+///
+/// This is an owning object reference, like [`Error`]. Payload reads through
+/// `interrupt.value` borrow the shared [`VisitInterruptObj`]; use
[`Self::new`]
+/// instead of a payload-only struct literal. Moving or replacing the payload
+/// through this reference is not supported; clone it to retain a separate
owner.
+#[repr(C)]
+#[derive(Clone, ObjectRef)]
+pub struct VisitInterrupt {
+ data: ObjectArc<VisitInterruptObj>,
+}
+
+impl VisitInterrupt {
+ /// Create an interrupt carrying an FFI-compatible payload back to the
caller of
+ /// the visit.
+ ///
+ /// # Arguments
+ ///
+ /// * `payload` - The payload, such as the value that was found.
+ ///
+ /// # Returns
+ ///
+ /// The interrupt.
+ #[inline(always)]
+ pub fn new<T: Into<Any>>(payload: T) -> Self {
+ Self::from_object(VisitInterruptObj {
+ base: Object::new(),
+ value: payload.into(),
+ })
+ }
}
/// Native state and default traversal for [`StructuralVisitEngine`].
@@ -202,83 +276,6 @@ pub struct StructuralVisitEngine<Base =
StructuralVisitorObj> {
dispatcher: *mut c_void,
}
-impl Default for StructuralVisitorObj {
- #[inline(always)]
- fn default() -> Self {
- Self {
- base: Object::new(),
- vtable: std::ptr::null(),
- def_region_mode: UnsafeCell::new(DefRegionKind::None as i32),
- }
- }
-}
-
-/// Definition-region state active at the current value.
-///
-/// Reflected fields marked `SEqHashDefPattern` or
-/// `SEqHashDefSimple` override the inherited state for that field's
-/// complete recursive visit.
-#[repr(i32)]
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
-pub enum DefRegionKind {
- /// The value is outside a definition region.
- #[default]
- None = 0,
- /// Definitions apply recursively through the visited value.
- Pattern = 1,
- /// Definitions apply to the visited value alone; its type is walked as
uses.
- Simple = 2,
-}
-
-const _: () = {
- assert!(DefRegionKind::None as i32 ==
TVMFFIDefRegionKind::kTVMFFIDefRegionKindNone as i32);
- assert!(
- DefRegionKind::Pattern as i32 ==
TVMFFIDefRegionKind::kTVMFFIDefRegionKindPattern as i32
- );
- assert!(DefRegionKind::Simple as i32 ==
TVMFFIDefRegionKind::kTVMFFIDefRegionKindSimple as i32);
-};
-
-/// Interrupt state of a traversal, mirroring C++ `ffi.VisitInterrupt`.
-///
-/// Entry points and visitor-layer calls return
-/// `Result<Option<VisitInterrupt>>`: `Ok(None)` means the (sub)graph was
-/// visited completely, `Ok(Some(..))` means a handler halted the traversal
-/// with this interrupt, and `Err` means it failed.
-///
-/// This is an owning object reference, like [`Error`]. Payload reads through
-/// `interrupt.value` borrow the shared [`VisitInterruptObj`]; use
[`Self::new`]
-/// instead of a payload-only struct literal. Moving or replacing the payload
-/// through this reference is not supported; clone it to retain a separate
owner.
-#[repr(C)]
-#[derive(Clone, ObjectRef)]
-pub struct VisitInterrupt {
- data: ObjectArc<VisitInterruptObj>,
-}
-
-impl VisitInterrupt {
- /// Create an interrupt carrying an FFI-compatible payload back to the
caller of
- /// the visit.
- ///
- /// # Arguments
- ///
- /// * `payload` - The payload, such as the value that was found.
- ///
- /// # Returns
- ///
- /// The interrupt.
- #[inline(always)]
- pub fn new<T: Into<Any>>(payload: T) -> Self {
- Self::from_object(VisitInterruptObj {
- base: Object::new(),
- value: payload.into(),
- })
- }
-}
-
-//
-----------------------------------------------------------------------------
-// Public visitor API
-//
-----------------------------------------------------------------------------
-
/// The running structural visit, passed to each handler.
///
/// A handler that matches a node decides which children are visited: call
@@ -295,6 +292,10 @@ pub struct StructuralVisitorView<'a, D, Base =
StructuralVisitorObj> {
_marker: PhantomData<(&'a mut (), fn(&mut D) -> Base, Rc<()>)>,
}
+//
-----------------------------------------------------------------------------
+// Public visitor API and entrypoint
+//
-----------------------------------------------------------------------------
+
impl<D, Base: StructuralVisitorBase> StructuralVisitorView<'_, D, Base> {
/// Inspect the base state; the borrow cannot outlive this view.
#[inline(always)]
@@ -417,6 +418,96 @@ impl<D, Base: StructuralVisitorBase>
StructuralVisitorView<'_, D, Base> {
}
}
+/// Visit `root` with an owned or borrowed [`Dispatcher`](crate::Dispatcher).
+///
+/// Handlers receive [`StructuralVisitorView`] for explicit recursion and keep
state on the
+/// dispatcher. Unmatched values use default child traversal. Pass `&mut D` to
+/// keep the dispatcher and inspect its state after the visit. An owned
dispatcher
+/// is held in the outer call and uses the same borrowed traversal engine.
+///
+/// Callbacks must report failures through [`Result`] and must not panic: a
panic
+/// reaching the non-unwinding native callback boundary aborts the process.
+///
+/// # Arguments
+///
+/// * `root` - Value borrowed for traversal, convertible into an [`AnyView`].
+/// * `dispatcher` - Owned or mutably borrowed dispatcher implementing
[`IntoDispatcher`].
+/// Matching handlers control recursion through [`StructuralVisitorView`];
+/// unmatched values use default child traversal.
+///
+/// # Examples
+///
+/// Sum integer leaves while default traversal descends into the array:
+///
+/// ```
+/// # fn main() -> tvm_ffi::Result<()> {
+/// use tvm_ffi::{
+/// dispatch, structural_visit, AnyView, Array, Result,
StructuralVisitorView,
+/// VisitInterrupt,
+/// };
+///
+/// struct Sum(i64);
+/// #[dispatch(visit, (AnyView<'_>, &mut StructuralVisitorView<'_, Self>),
Result<Option<VisitInterrupt>>)]
+/// impl Sum {
+/// fn visit_integer(
+/// &mut self,
+/// value: i64,
+/// _: &mut StructuralVisitorView<'_, Self>,
+/// ) -> Result<Option<VisitInterrupt>> {
+/// self.0 += value;
+/// Ok(None)
+/// }
+/// }
+///
+/// let root = Array::from_vec(vec![1_i64, 2, 3]);
+/// let mut sum = Sum(0);
+/// assert!(structural_visit(&root, &mut sum)?.is_none());
+/// assert_eq!(sum.0, 6);
+/// # Ok(())
+/// # }
+/// ```
+pub fn structural_visit<'input, R, H, M, D, O>(
+ root: R,
+ dispatcher: H,
+) -> Result<Option<VisitInterrupt>>
+where
+ R: Into<AnyView<'input>>,
+ O: BorrowMut<D>,
+ for<'v, 'c, 'r> H: IntoDispatcher<
+ (
+ AnyView<'v>,
+ &'c mut StructuralVisitorView<'r, D, StructuralVisitorObj>,
+ ),
+ Option<Result<Option<VisitInterrupt>>>,
+ M,
+ Target = D,
+ Output = O,
+ >,
+ for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
+ (
+ AnyView<'v>,
+ &'c mut StructuralVisitorView<'r, D, StructuralVisitorObj>,
+ ),
+ Option<Result<Option<VisitInterrupt>>>,
+ >,
+{
+ StructuralVisitEngine::visit_root(root, dispatcher)
+}
+
+impl super::dispatch::CallbackResult<Result<Option<VisitInterrupt>>>
+ for Result<Option<VisitInterrupt>>
+{
+ #[inline(always)]
+ fn callback_result(self) -> Result<Option<VisitInterrupt>> {
+ self
+ }
+}
+
+//
-----------------------------------------------------------------------------
+// StructuralVisitEngine implementation
+//
-----------------------------------------------------------------------------
+
+// Generic runtime: only this implementation projects the dispatcher after
Base.
impl<Base: StructuralVisitorBase> StructuralVisitEngine<Base> {
/// Visit `root` with `dispatcher` on the native base `Base`. See also
[`structural_visit`].
#[inline(always)]
@@ -445,121 +536,184 @@ impl<Base: StructuralVisitorBase>
StructuralVisitEngine<Base> {
let root = root.into();
Self::run_visitor::<D>(*root.as_raw_ffi_any(), storage.borrow_mut())
}
-}
-impl super::dispatch::CallbackResult<Result<Option<VisitInterrupt>>>
- for Result<Option<VisitInterrupt>>
-{
- #[inline(always)]
- fn callback_result(self) -> Result<Option<VisitInterrupt>> {
- self
- }
-}
-
-//
-----------------------------------------------------------------------------
-// Outlined errors
-//
-----------------------------------------------------------------------------
-
-#[cold]
-#[inline(never)]
-fn unregistered_visit_type_error(type_index: i32) -> Error {
- runtime_error(&format!(
- "native visitor: unregistered type index {type_index}"
- ))
-}
-
-#[cold]
-#[inline(never)]
-fn invalid_visit_hook_result_error() -> Error {
- Error::new(
- TYPE_ERROR,
- "structural visit hook must return None or ffi.VisitInterrupt",
- "",
- )
-}
-
-/// Consume `error` and annotate it from borrowed input scalar fields. Internal
-/// callers keep that input live; the Error remains owned if formatting
unwinds.
-// Perf note: Keep error formatting and context-allocation scratch out of
visit frames.
-#[cold]
-#[inline(never)]
-fn annotate_visit_error(error: Error, value: TVMFFIAny) -> Error {
- let error = update_visit_error_context(error, value);
- with_structural_error_context(
- error,
- "visit",
- &format!(
- "object `{}`",
- crate::object::type_key_of_index(value.type_index)
- ),
- )
-}
-
-//
-----------------------------------------------------------------------------
-// Private utility functions
-//
-----------------------------------------------------------------------------
+ /// The engine whose first field is the native prefix `runtime`.
+ ///
+ /// # Safety
+ /// `runtime` must belong to the live, synchronous Rust traversal that
created
+ /// this exact repr(C) engine. An arbitrary native visitor is not such an
engine.
+ #[inline(always)]
+ unsafe fn from_handle(runtime: StructuralVisitorHandle) -> *mut Self {
+ runtime.cast()
+ }
-#[inline(always)]
-fn dispatcher_ptr<D>(dispatch: &mut D) -> *mut c_void {
- std::ptr::from_mut(dispatch).cast()
-}
+ /// Publish the current exclusive `dispatch` reborrow and visit borrowed
`raw`.
+ /// Return one owning raw None/interrupt/Error token.
+ ///
+ /// # Safety
+ /// `runtime` must belong to this live `D` invocation and `raw` remain
valid
+ /// through recursion; the caller cannot use competing dispatcher borrows.
+ #[inline(always)]
+ unsafe fn visit_raw<D>(
+ runtime: StructuralVisitorHandle,
+ dispatch: &mut D,
+ raw: TVMFFIAny,
+ ) -> TVMFFIAny {
+ let context = Self::checked_dispatcher(runtime, dispatch);
+ (*Self::from_handle(runtime)).dispatcher = context;
+ StructuralVisitorObj::call_visitor_raw(runtime, raw)
+ }
-// The visit result carrier, as C++: None continues; a VisitInterrupt or an
Error stops.
-#[inline(always)]
-fn visit_result_raw(result: Result<Option<VisitInterrupt>>, raw: TVMFFIAny) ->
TVMFFIAny {
- match result {
- Ok(None) => TVMFFIAny::new(),
- Ok(Some(interrupt)) => Any::into_raw_ffi_any(interrupt.into()),
- Err(error) => {
+ /// Check runtime identity and return the current `dispatch` reborrow
pointer.
+ ///
+ /// # Safety
+ /// `runtime` must be the live engine for `D`; address equality does not
establish
+ /// type or lifetime validity. Mismatch panics before a pointer is
published.
+ // The caller has already established matching runtime/D types and scope.
+ // Read only the stored raw address: do not create a competing mutable
borrow.
+ #[inline(always)]
+ unsafe fn checked_dispatcher<D>(
+ runtime: StructuralVisitorHandle,
+ dispatch: &mut D,
+ ) -> *mut c_void {
+ let current = dispatcher_ptr(dispatch);
+ if current != (*Self::from_handle(runtime)).dispatcher {
core::hint::cold_path();
-
StructuralVisitorObj::with_visit_context_raw(Any::into_raw_ffi_any(error.into()),
raw)
+ wrong_dispatcher();
}
+ current
}
-}
-// A packed hook's owning Any is valid, but its result type is dynamic.
-#[inline(always)]
-fn checked_hook_result(value: Any) -> Any {
- let tag = value.type_index();
- if tag == TVMFFITypeIndex::kTVMFFINone as i32
- || tag == TVMFFITypeIndex::kTVMFFIError as i32
- || tag == TVMFFITypeIndex::kTVMFFIVisitInterrupt as i32
+ /// Build the native visitor callback for concrete dispatcher `D`. Its
higher-ranked
+ /// view arguments restrict all borrows to the synchronous handler
invocation.
+ #[inline(always)]
+ fn vtable<D>() -> &'static StructuralVisitorVTable
+ where
+ for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
+ (AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
+ Option<Result<Option<VisitInterrupt>>>,
+ >,
{
- value
- } else {
- invalid_visit_hook_result_error().into()
+ &StructuralVisitorVTable {
+ visit: Self::rust_vtable_visit_raw::<D>,
+ }
}
-}
-/// Consume exactly one Error reference after its integer tag was checked.
-#[inline(always)]
-unsafe fn take_error(value: Any) -> Error {
- use crate::type_traits::AnyCompatible;
- let mut raw = Any::into_raw_ffi_any(value);
- Error::move_from_any_after_check(&mut raw)
-}
-
-#[inline(always)]
-fn with_field_error_context(result: Any, name: &str) -> Any {
- if result.type_index() == TVMFFITypeIndex::kTVMFFIError as i32 {
- core::hint::cold_path();
- // SAFETY: direct tag check establishes the Error representation.
- with_structural_field_error_context(unsafe { take_error(result) },
"visit", name).into()
- } else {
+ /// Enter a native visitor callback with borrowed `value`; return one
owning token.
+ /// Restore the incoming dispatcher provenance before returning to the
caller.
+ ///
+ /// # Safety
+ /// `runtime` must have this `D` vtable and a live published reborrow. The
erased
+ /// ABI lifetime on `value` is valid only for this call. Callback failures
use
+ /// Result; Rust panics cannot cross this native C boundary.
+ unsafe extern "C" fn rust_vtable_visit_raw<D>(
+ runtime: StructuralVisitorHandle,
+ value: AnyView<'static>,
+ ) -> TVMFFIAny
+ where
+ for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
+ (AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
+ Option<Result<Option<VisitInterrupt>>>,
+ >,
+ {
+ let context = (*Self::from_handle(runtime)).dispatcher;
+ let raw = *value.as_raw_ffi_any();
+ let result = if raw.type_index == TVMFFITypeIndex::kTVMFFINone as i32 {
+ core::hint::cold_path();
+ TVMFFIAny::new()
+ } else {
+ let dispatch = &mut *context.cast::<D>();
+ let mut visitor = Self::visitor_context::<D>(runtime);
+ // SAFETY: the native caller keeps the input alive for this
synchronous
+ // dispatch; the higher-ranked callback cannot retain its borrowed
view.
+ match crate::Dispatcher::dispatch_tuple(
+ &mut *dispatch,
+ (AnyView::from_raw_ffi_any(raw), &mut visitor),
+ ) {
+ Some(result) => visit_result_raw(result, raw),
+ None => Self::default_visit_raw(runtime,
dispatcher_ptr(dispatch), raw),
+ }
+ };
+ // The enclosing callback may resume using its ancestor mutable borrow.
+ // Native hooks cannot restore Rust-only borrow provenance for us.
+ (*Self::from_handle(runtime)).dispatcher = context;
result
}
-}
-// Columns are process-lifetime registry entries; a missing per-type cell still
-// falls back to reflected traversal. A missing required column is an error.
-static STRUCTURAL_VISIT_COLUMN: ColdInitLazyLock<TypeAttrColumn> =
ColdInitLazyLock::new(|| {
- TypeAttrColumn::new(STRUCTURAL_VISIT_ATTR).expect("missing __s_visit__
column")
-});
+ /// Create the handler view for borrowed `runtime`, without retaining it.
+ ///
+ /// # Safety
+ /// The live runtime's vtable must match D. Keep this view within the
current
+ /// higher-ranked handler call, never outliving the runtime or dispatcher.
+ #[inline(always)]
+ unsafe fn visitor_context<'a, D>(
+ runtime: StructuralVisitorHandle,
+ ) -> StructuralVisitorView<'a, D, Base> {
+ StructuralVisitorView {
+ runtime,
+ _marker: PhantomData,
+ }
+ }
-/// Return the shared visit-hook column without caching optional per-type
cells.
-#[inline(always)]
-fn structural_visit_column() -> TypeAttrColumn {
- *STRUCTURAL_VISIT_COLUMN
+ /// Publish `context` and descend through borrowed `raw`, then annotate
errors.
+ /// The raw return transfers one owning None/interrupt/Error token.
+ ///
+ /// # Safety
+ /// `runtime` must be live and `context` its current exclusive dispatcher
reborrow;
+ /// `raw` remains borrowed and must survive the complete synchronous
descent.
+ #[inline(always)]
+ unsafe fn default_visit_raw(
+ runtime: StructuralVisitorHandle,
+ context: *mut c_void,
+ raw: TVMFFIAny,
+ ) -> TVMFFIAny {
+ // Context is the supplied exclusive reborrow. Publish it before
descent;
+ // the handle-only scanner never creates or uses a competing D
reference.
+ (*Self::from_handle(runtime)).dispatcher = context;
+ StructuralVisitorObj::with_visit_context_raw(
+ Base::default_visit_raw(runtime.cast(),
AnyView::from_raw_ffi_any(raw)),
+ raw,
+ )
+ }
+
+ /// Visit borrowed `root` with `dispatch` on a fresh engine; the root's
owner outlives it.
+ pub(super) fn run_visitor<D>(
+ root: TVMFFIAny,
+ dispatch: &mut D,
+ ) -> Result<Option<VisitInterrupt>>
+ where
+ for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
+ (AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
+ Option<Result<Option<VisitInterrupt>>>,
+ >,
+ {
+ let base = Base::new()?;
+ let type_index = Base::type_index();
+ // The managed runtime may borrow its dispatcher only for this call.
+ // Trusted hooks release all temporary handles before the owner is
dropped.
+ let mut engine = unsafe {
+ ObjectArc::<Object>::new_opaque_unchecked(
+ StructuralVisitEngine {
+ base,
+ dispatcher: dispatcher_ptr(dispatch),
+ },
+ type_index,
+ )
+ };
+ let runtime = ObjectArc::as_mut_ptr(&mut
engine).cast::<StructuralVisitorObj>();
+ // The engine and outer dispatcher storage stay stationary for the
whole run.
+ // Only raw field projections cross recursion; no whole-engine borrow
survives.
+ unsafe {
+ (*runtime).vtable = Self::vtable::<D>();
+ *(*runtime).def_region_mode.get() = DefRegionKind::None as i32;
+ }
+ // Keep the result owned while releasing the runtime, including on
unwind.
+ let result =
+ unsafe {
Any::from_raw_ffi_any(StructuralVisitorObj::call_visitor_raw(runtime, root)) };
+ // Trusted hooks have released every temporary strong/weak runtime
reference.
+ drop(engine);
+ visit_result_from_any(result)
+ }
}
//
-----------------------------------------------------------------------------
@@ -570,12 +724,26 @@ impl StructuralVisitorObj {
/// Consume `result`, naming the borrowed `value` in it when it is an
Error.
#[inline(always)]
fn with_visit_context_raw(result: TVMFFIAny, value: TVMFFIAny) ->
TVMFFIAny {
+ if result.type_index == TVMFFITypeIndex::kTVMFFINone as i32 {
+ result
+ } else {
+ Self::with_visit_stop_context_raw(result, value)
+ }
+ }
+
+ // Continuing visits need no owning conversion or input inspection. Keep
+ // interrupt/error handling out of their recursive frames.
+ #[cold]
+ #[inline(never)]
+ extern "C-unwind" fn with_visit_stop_context_raw(
+ result: TVMFFIAny,
+ value: TVMFFIAny,
+ ) -> TVMFFIAny {
if result.type_index != TVMFFITypeIndex::kTVMFFIError as i32
|| value.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as
i32
{
result
} else {
- core::hint::cold_path();
// SAFETY: the Error tag establishes the owned object's
representation.
unsafe {
let error = take_error(Any::from_raw_ffi_any(result));
@@ -744,297 +912,58 @@ impl StructuralVisitorObj {
Any::into_raw_ffi_any(result)
}
- /// Run `callback` under requested `kind` on live borrowed `visitor`,
preserving
- /// Pattern precedence. Restore prior region on return or Rust unwind.
Internal
- /// callers keep the runtime live and retain no reference to its region
field.
- #[inline(always)]
- fn with_def_region_kind<T>(
- visitor: StructuralVisitorHandle,
- kind: DefRegionKind,
- callback: impl FnOnce() -> T,
- ) -> T {
- unsafe {
- let previous = *(*visitor).def_region_mode.get();
- if previous == kind as i32 || previous == DefRegionKind::Pattern
as i32 {
- // Inherited descent does not change state; Pattern keeps
precedence.
- return callback();
- }
- *(*visitor).def_region_mode.get() = kind as i32;
- struct Restore {
- visitor: StructuralVisitorHandle,
- previous: i32,
- }
- impl Drop for Restore {
- #[inline(always)]
- fn drop(&mut self) {
- unsafe { *(*self.visitor).def_region_mode.get() =
self.previous };
- }
- }
- let _restore = Restore { visitor, previous };
- callback()
- }
- }
-}
-
-// Generic runtime: only this implementation projects the dispatcher after
Base.
-impl<Base: StructuralVisitorBase> StructuralVisitEngine<Base> {
- /// The engine whose first field is the native prefix `runtime`.
- ///
- /// # Safety
- /// `runtime` must belong to the live, synchronous Rust traversal that
created
- /// this exact repr(C) engine. An arbitrary native visitor is not such an
engine.
- #[inline(always)]
- unsafe fn from_handle(runtime: StructuralVisitorHandle) -> *mut Self {
- runtime.cast()
- }
-
- /// Publish the current exclusive `dispatch` reborrow and visit borrowed
`raw`.
- /// Return one owning raw None/interrupt/Error token.
- ///
- /// # Safety
- /// `runtime` must belong to this live `D` invocation and `raw` remain
valid
- /// through recursion; the caller cannot use competing dispatcher borrows.
- #[inline(always)]
- unsafe fn visit_raw<D>(
- runtime: StructuralVisitorHandle,
- dispatch: &mut D,
- raw: TVMFFIAny,
- ) -> TVMFFIAny {
- let context = Self::checked_dispatcher(runtime, dispatch);
- (*Self::from_handle(runtime)).dispatcher = context;
- StructuralVisitorObj::call_visitor_raw(runtime, raw)
- }
-
- /// Check runtime identity and return the current `dispatch` reborrow
pointer.
- ///
- /// # Safety
- /// `runtime` must be the live engine for `D`; address equality does not
establish
- /// type or lifetime validity. Mismatch panics before a pointer is
published.
- // The caller has already established matching runtime/D types and scope.
- // Read only the stored raw address: do not create a competing mutable
borrow.
- #[inline(always)]
- unsafe fn checked_dispatcher<D>(
- runtime: StructuralVisitorHandle,
- dispatch: &mut D,
- ) -> *mut c_void {
- let current = dispatcher_ptr(dispatch);
- if current != (*Self::from_handle(runtime)).dispatcher {
- core::hint::cold_path();
- wrong_dispatcher();
- }
- current
- }
-
- /// Build the native visitor callback for concrete dispatcher `D`. Its
higher-ranked
- /// view arguments restrict all borrows to the synchronous handler
invocation.
- #[inline(always)]
- fn vtable<D>() -> &'static StructuralVisitorVTable
- where
- for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
- (AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
- Option<Result<Option<VisitInterrupt>>>,
- >,
- {
- &StructuralVisitorVTable {
- visit: Self::rust_vtable_visit_raw::<D>,
- }
- }
-
- /// Enter a native visitor callback with borrowed `value`; return one
owning token.
- /// Restore the incoming dispatcher provenance before returning to the
caller.
- ///
- /// # Safety
- /// `runtime` must have this `D` vtable and a live published reborrow. The
erased
- /// ABI lifetime on `value` is valid only for this call. Callback failures
use
- /// Result; Rust panics cannot cross this native C boundary.
- unsafe extern "C" fn rust_vtable_visit_raw<D>(
- runtime: StructuralVisitorHandle,
- value: AnyView<'static>,
- ) -> TVMFFIAny
- where
- for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
- (AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
- Option<Result<Option<VisitInterrupt>>>,
- >,
- {
- let context = (*Self::from_handle(runtime)).dispatcher;
- let raw = *value.as_raw_ffi_any();
- let result = if raw.type_index == TVMFFITypeIndex::kTVMFFINone as i32 {
- core::hint::cold_path();
- TVMFFIAny::new()
- } else {
- let dispatch = &mut *context.cast::<D>();
- let mut visitor = Self::visitor_context::<D>(runtime);
- // SAFETY: the native caller keeps the input alive for this
synchronous
- // dispatch; the higher-ranked callback cannot retain its borrowed
view.
- match crate::Dispatcher::dispatch_tuple(
- &mut *dispatch,
- (AnyView::from_raw_ffi_any(raw), &mut visitor),
- ) {
- Some(result) => visit_result_raw(result, raw),
- None => Self::default_visit_raw(runtime,
dispatcher_ptr(dispatch), raw),
- }
- };
- // The enclosing callback may resume using its ancestor mutable borrow.
- // Native hooks cannot restore Rust-only borrow provenance for us.
- (*Self::from_handle(runtime)).dispatcher = context;
- result
- }
-
- /// Create the handler view for borrowed `runtime`, without retaining it.
- ///
- /// # Safety
- /// The live runtime's vtable must match D. Keep this view within the
current
- /// higher-ranked handler call, never outliving the runtime or dispatcher.
- #[inline(always)]
- unsafe fn visitor_context<'a, D>(
- runtime: StructuralVisitorHandle,
- ) -> StructuralVisitorView<'a, D, Base> {
- StructuralVisitorView {
- runtime,
- _marker: PhantomData,
- }
- }
-
- /// Publish `context` and descend through borrowed `raw`, then annotate
errors.
- /// The raw return transfers one owning None/interrupt/Error token.
- ///
- /// # Safety
- /// `runtime` must be live and `context` its current exclusive dispatcher
reborrow;
- /// `raw` remains borrowed and must survive the complete synchronous
descent.
+ /// Run `callback` under requested `kind` on live borrowed `visitor`,
preserving
+ /// Pattern precedence. Restore prior region on return or Rust unwind.
Internal
+ /// callers keep the runtime live and retain no reference to its region
field.
#[inline(always)]
- unsafe fn default_visit_raw(
- runtime: StructuralVisitorHandle,
- context: *mut c_void,
- raw: TVMFFIAny,
- ) -> TVMFFIAny {
- // Context is the supplied exclusive reborrow. Publish it before
descent;
- // the handle-only scanner never creates or uses a competing D
reference.
- (*Self::from_handle(runtime)).dispatcher = context;
- StructuralVisitorObj::with_visit_context_raw(
- Base::default_visit_raw(runtime.cast(),
AnyView::from_raw_ffi_any(raw)),
- raw,
- )
- }
-
- /// Visit borrowed `root` with `dispatch` on a fresh engine; the root's
owner outlives it.
- pub(super) fn run_visitor<D>(
- root: TVMFFIAny,
- dispatch: &mut D,
- ) -> Result<Option<VisitInterrupt>>
- where
- for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
- (AnyView<'v>, &'c mut StructuralVisitorView<'r, D, Base>),
- Option<Result<Option<VisitInterrupt>>>,
- >,
- {
- let base = Base::new()?;
- let type_index = Base::type_index();
- // The managed runtime may borrow its dispatcher only for this call.
- // Trusted hooks release all temporary handles before the owner is
dropped.
- let mut engine = unsafe {
- ObjectArc::<Object>::new_opaque_unchecked(
- StructuralVisitEngine {
- base,
- dispatcher: dispatcher_ptr(dispatch),
- },
- type_index,
- )
- };
- let runtime = ObjectArc::as_mut_ptr(&mut
engine).cast::<StructuralVisitorObj>();
- // The engine and outer dispatcher storage stay stationary for the
whole run.
- // Only raw field projections cross recursion; no whole-engine borrow
survives.
+ fn with_def_region_kind<T>(
+ visitor: StructuralVisitorHandle,
+ kind: DefRegionKind,
+ callback: impl FnOnce() -> T,
+ ) -> T {
unsafe {
- (*runtime).vtable = Self::vtable::<D>();
- *(*runtime).def_region_mode.get() = DefRegionKind::None as i32;
+ let previous = *(*visitor).def_region_mode.get();
+ if previous == kind as i32 || previous == DefRegionKind::Pattern
as i32 {
+ // Inherited descent does not change state; Pattern keeps
precedence.
+ return callback();
+ }
+ *(*visitor).def_region_mode.get() = kind as i32;
+ struct Restore {
+ visitor: StructuralVisitorHandle,
+ previous: i32,
+ }
+ impl Drop for Restore {
+ #[inline(always)]
+ fn drop(&mut self) {
+ unsafe { *(*self.visitor).def_region_mode.get() =
self.previous };
+ }
+ }
+ let _restore = Restore { visitor, previous };
+ callback()
}
- // Keep the result owned while releasing the runtime, including on
unwind.
- let result =
- unsafe {
Any::from_raw_ffi_any(StructuralVisitorObj::call_visitor_raw(runtime, root)) };
- // Trusted hooks have released every temporary strong/weak runtime
reference.
- drop(engine);
- visit_result_from_any(result)
}
}
//
-----------------------------------------------------------------------------
-// Public entrypoint
+// Private utility functions
//
-----------------------------------------------------------------------------
-/// Visit `root` with an owned or borrowed [`Dispatcher`](crate::Dispatcher).
-///
-/// Handlers receive [`StructuralVisitorView`] for explicit recursion and keep
state on the
-/// dispatcher. Unmatched values use default child traversal. Pass `&mut D` to
-/// keep the dispatcher and inspect its state after the visit. An owned
dispatcher
-/// is held in the outer call and uses the same borrowed traversal engine.
-///
-/// Callbacks must report failures through [`Result`] and must not panic: a
panic
-/// reaching the non-unwinding native callback boundary aborts the process.
-///
-/// # Arguments
-///
-/// * `root` - Value borrowed for traversal, convertible into an [`AnyView`].
-/// * `dispatcher` - Owned or mutably borrowed dispatcher implementing
[`IntoDispatcher`].
-/// Matching handlers control recursion through [`StructuralVisitorView`];
-/// unmatched values use default child traversal.
-///
-/// # Examples
-///
-/// Sum integer leaves while default traversal descends into the array:
-///
-/// ```
-/// # fn main() -> tvm_ffi::Result<()> {
-/// use tvm_ffi::{
-/// dispatch, structural_visit, AnyView, Array, Result,
StructuralVisitorView,
-/// VisitInterrupt,
-/// };
-///
-/// struct Sum(i64);
-/// #[dispatch(visit, (AnyView<'_>, &mut StructuralVisitorView<'_, Self>),
Result<Option<VisitInterrupt>>)]
-/// impl Sum {
-/// fn visit_integer(
-/// &mut self,
-/// value: i64,
-/// _: &mut StructuralVisitorView<'_, Self>,
-/// ) -> Result<Option<VisitInterrupt>> {
-/// self.0 += value;
-/// Ok(None)
-/// }
-/// }
-///
-/// let root = Array::from_vec(vec![1_i64, 2, 3]);
-/// let mut sum = Sum(0);
-/// assert!(structural_visit(&root, &mut sum)?.is_none());
-/// assert_eq!(sum.0, 6);
-/// # Ok(())
-/// # }
-/// ```
-pub fn structural_visit<'input, R, H, M, D, O>(
- root: R,
- dispatcher: H,
-) -> Result<Option<VisitInterrupt>>
-where
- R: Into<AnyView<'input>>,
- O: BorrowMut<D>,
- for<'v, 'c, 'r> H: IntoDispatcher<
- (
- AnyView<'v>,
- &'c mut StructuralVisitorView<'r, D, StructuralVisitorObj>,
- ),
- Option<Result<Option<VisitInterrupt>>>,
- M,
- Target = D,
- Output = O,
- >,
- for<'s, 'v, 'c, 'r> &'s mut D: crate::Dispatcher<
- (
- AnyView<'v>,
- &'c mut StructuralVisitorView<'r, D, StructuralVisitorObj>,
- ),
- Option<Result<Option<VisitInterrupt>>>,
- >,
-{
- StructuralVisitEngine::visit_root(root, dispatcher)
+#[inline(always)]
+fn dispatcher_ptr<D>(dispatch: &mut D) -> *mut c_void {
+ std::ptr::from_mut(dispatch).cast()
+}
+
+// The visit result carrier, as C++: None continues; a VisitInterrupt or an
Error stops.
+#[inline(always)]
+fn visit_result_raw(result: Result<Option<VisitInterrupt>>, raw: TVMFFIAny) ->
TVMFFIAny {
+ match result {
+ Ok(None) => TVMFFIAny::new(),
+ Ok(Some(interrupt)) => Any::into_raw_ffi_any(interrupt.into()),
+ Err(error) => {
+ core::hint::cold_path();
+
StructuralVisitorObj::with_visit_context_raw(Any::into_raw_ffi_any(error.into()),
raw)
+ }
+ }
}
#[inline(always)]
@@ -1057,3 +986,87 @@ pub(super) fn visit_result_from_any(result: Any) ->
Result<Option<VisitInterrupt
}))
}
}
+
+// A packed hook's owning Any is valid, but its result type is dynamic.
+#[inline(always)]
+fn checked_hook_result(value: Any) -> Any {
+ let tag = value.type_index();
+ if tag == TVMFFITypeIndex::kTVMFFINone as i32
+ || tag == TVMFFITypeIndex::kTVMFFIError as i32
+ || tag == TVMFFITypeIndex::kTVMFFIVisitInterrupt as i32
+ {
+ value
+ } else {
+ invalid_visit_hook_result_error().into()
+ }
+}
+
+/// Consume exactly one Error reference after its integer tag was checked.
+#[inline(always)]
+unsafe fn take_error(value: Any) -> Error {
+ use crate::type_traits::AnyCompatible;
+ let mut raw = Any::into_raw_ffi_any(value);
+ Error::move_from_any_after_check(&mut raw)
+}
+
+#[inline(always)]
+fn with_field_error_context(result: Any, name: &str) -> Any {
+ if result.type_index() == TVMFFITypeIndex::kTVMFFIError as i32 {
+ core::hint::cold_path();
+ // SAFETY: direct tag check establishes the Error representation.
+ with_structural_field_error_context(unsafe { take_error(result) },
"visit", name).into()
+ } else {
+ result
+ }
+}
+
+// Columns are process-lifetime registry entries; a missing per-type cell still
+// falls back to reflected traversal. A missing required column is an error.
+static STRUCTURAL_VISIT_COLUMN: ColdInitLazyLock<TypeAttrColumn> =
ColdInitLazyLock::new(|| {
+ TypeAttrColumn::new(STRUCTURAL_VISIT_ATTR).expect("missing __s_visit__
column")
+});
+
+/// Return the shared visit-hook column without caching optional per-type
cells.
+#[inline(always)]
+fn structural_visit_column() -> TypeAttrColumn {
+ *STRUCTURAL_VISIT_COLUMN
+}
+
+//
-----------------------------------------------------------------------------
+// Outlined errors
+//
-----------------------------------------------------------------------------
+
+#[cold]
+#[inline(never)]
+fn unregistered_visit_type_error(type_index: i32) -> Error {
+ runtime_error(&format!(
+ "native visitor: unregistered type index {type_index}"
+ ))
+}
+
+#[cold]
+#[inline(never)]
+fn invalid_visit_hook_result_error() -> Error {
+ Error::new(
+ TYPE_ERROR,
+ "structural visit hook must return None or ffi.VisitInterrupt",
+ "",
+ )
+}
+
+/// Consume `error` and annotate it from borrowed input scalar fields. Internal
+/// callers keep that input live; the Error remains owned if formatting
unwinds.
+// Perf note: Keep error formatting and context-allocation scratch out of
visit frames.
+#[cold]
+#[inline(never)]
+fn annotate_visit_error(error: Error, value: TVMFFIAny) -> Error {
+ let error = update_visit_error_context(error, value);
+ with_structural_error_context(
+ error,
+ "visit",
+ &format!(
+ "object `{}`",
+ crate::object::type_key_of_index(value.type_index)
+ ),
+ )
+}
diff --git a/rust/tvm-ffi/src/lib.rs b/rust/tvm-ffi/src/lib.rs
index c3672c82..880b6023 100644
--- a/rust/tvm-ffi/src/lib.rs
+++ b/rust/tvm-ffi/src/lib.rs
@@ -93,7 +93,8 @@ pub use crate::type_traits::AnyCompatible;
/// `None` when no handler matches. The macro also implements [`Dispatcher`]
/// for `&Self` or `&mut Self`, so the type can drive [`structural_walk`] and
/// the other traversals, whose engines call it and run their own default
-/// recursion; no prefix has a reserved meaning.
+/// recursion; no prefix has a reserved meaning. Handlers without an `inline`
+/// attribute are marked `#[inline(always)]`.
///
/// # Arguments
///
diff --git a/rust/tvm-ffi/src/macros.rs b/rust/tvm-ffi/src/macros.rs
index c87d65f3..206ec201 100644
--- a/rust/tvm-ffi/src/macros.rs
+++ b/rust/tvm-ffi/src/macros.rs
@@ -19,6 +19,7 @@
// Re-export paste for the export macros, so downstream crates need not depend
on it.
#[doc(hidden)]
pub use paste;
+
// ----------------------------------------------------------------------------
// Macros for error handling
// ----------------------------------------------------------------------------
@@ -74,6 +75,166 @@ macro_rules! cached_global_func {
}};
}
+/// Check the return code of the safe call
+///
+/// # Arguments
+/// * `ret_code` - The return code of the safe call
+///
+/// # Returns
+/// * `Result<(), Error>` - The result of the safe call
+/// Macro to check safe calls and automatically update traceback with
file/line info
+///
+/// Usage: check_safe_call!(function(args))?;
+#[doc(hidden)]
+#[macro_export]
+macro_rules! check_safe_call {
+ ($expr:expr) => {{
+ let ret_code = $expr;
+ if ret_code == 0 {
+ Ok(())
+ } else {
+ let error = $crate::error::Error::from_raised();
+ Err(error)
+ }
+ }};
+}
+
+/// Return an [`Error`](crate::Error) of a kind with a formatted message from
+/// the current function.
+///
+/// `bail!(kind, fmt, args...)` formats the message as `format!` does and
+/// records the source location in the backtrace.
+///
+/// # Arguments
+///
+/// * `kind` - The kind of the error, such as
[`VALUE_ERROR`](crate::VALUE_ERROR).
+/// * `fmt` - The format string of the message.
+/// * `args` - The format arguments.
+///
+/// # Returns
+///
+/// Returns `Err` of the error from the enclosing function.
+///
+/// # Examples
+///
+/// ```
+/// use tvm_ffi::{bail, Result, INDEX_ERROR};
+///
+/// fn get(items: &[i64], index: usize) -> Result<i64> {
+/// if index >= items.len() {
+/// bail!(INDEX_ERROR, "index {} out of range for length {}", index,
items.len());
+/// }
+/// Ok(items[index])
+/// }
+///
+/// assert_eq!(get(&[1, 2], 1)?, 2);
+/// assert_eq!(get(&[1, 2], 5).unwrap_err().message(), "index 5 out of range
for length 2");
+/// # Ok::<(), tvm_ffi::Error>(())
+/// ```
+#[macro_export]
+macro_rules! bail {
+ ($error_kind:expr, $fmt:expr $(, $args:expr)* $(,)?) => {{
+ let context = format!(
+ " File \"{}\", line {}, in {}\n",
+ file!(),
+ line!(),
+ $crate::function_name!()
+ );
+ return Err($crate::error::Error::new($error_kind, &format!($fmt $(,
$args)*), &context));
+ }};
+}
+
+/// Return an [`Error`](crate::Error) from the current function unless a
+/// condition holds.
+///
+/// `ensure!(cond, kind, fmt, args...)` is [`bail!`](crate::bail)`(kind, fmt,
+/// args...)` when `cond` is false.
+///
+/// # Arguments
+///
+/// * `cond` - The condition that must hold.
+/// * `kind` - The kind of the error.
+/// * `fmt` - The format string of the message.
+/// * `args` - The format arguments.
+///
+/// # Returns
+///
+/// Returns `Err` of the error from the enclosing function when `cond` is
+/// false, and does nothing otherwise.
+///
+/// # Examples
+///
+/// ```
+/// use tvm_ffi::{ensure, Result, VALUE_ERROR};
+///
+/// fn sqrt(x: f64) -> Result<f64> {
+/// ensure!(x >= 0.0, VALUE_ERROR, "sqrt of negative number {}", x);
+/// Ok(x.sqrt())
+/// }
+///
+/// assert_eq!(sqrt(4.0)?, 2.0);
+/// assert!(sqrt(-1.0).is_err());
+/// # Ok::<(), tvm_ffi::Error>(())
+/// ```
+#[macro_export]
+macro_rules! ensure {
+ ($cond:expr, $error_kind:expr, $fmt:expr $(, $args:expr)* $(,)?) => {{
+ if !$cond {
+ $crate::bail!($error_kind, $fmt $(, $args)*);
+ }
+ }};
+}
+
+/// Append the current source location to the backtrace of an `Err` result,
+/// and pass an `Ok` through.
+///
+/// # Arguments
+///
+/// * `result` - The result to attach the context to.
+///
+/// # Returns
+///
+/// The result, with the location appended to an error's backtrace.
+///
+/// # Examples
+///
+/// ```
+/// use tvm_ffi::{attach_context, bail, Result, RUNTIME_ERROR};
+///
+/// fn inner() -> Result<()> {
+/// bail!(RUNTIME_ERROR, "failed");
+/// }
+///
+/// fn outer() -> Result<()> {
+/// attach_context!(inner())
+/// }
+///
+/// assert!(outer().unwrap_err().backtrace().contains("outer"));
+/// ```
+#[macro_export]
+macro_rules! attach_context {
+ ($error:expr) => {{
+ match $error {
+ Ok(value) => Ok(value),
+ Err(error) => {
+ let context = format!(
+ " File \"{}\", line {}, in {}\n",
+ file!(),
+ line!(),
+ $crate::function_name!()
+ );
+ Err($crate::error::Error::with_appended_backtrace(
+ error, &context,
+ ))
+ }
+ }
+ }};
+}
+
+// ----------------------------------------------------------------------------
+// Macros for object references
+// ----------------------------------------------------------------------------
+
/// Implement zero-copy conversions from a derived object reference to one of
/// its base object-reference types: `From<Source>` and `From<&Source>` into
/// `Target`, `AsRef<Target>`, which borrows a `&Source` as `&Target`
@@ -288,162 +449,6 @@ macro_rules! unsafe_impl_object_ref_downcast_ref {
};
}
-/// Check the return code of the safe call
-///
-/// # Arguments
-/// * `ret_code` - The return code of the safe call
-///
-/// # Returns
-/// * `Result<(), Error>` - The result of the safe call
-/// Macro to check safe calls and automatically update traceback with
file/line info
-///
-/// Usage: check_safe_call!(function(args))?;
-#[doc(hidden)]
-#[macro_export]
-macro_rules! check_safe_call {
- ($expr:expr) => {{
- let ret_code = $expr;
- if ret_code == 0 {
- Ok(())
- } else {
- let error = $crate::error::Error::from_raised();
- Err(error)
- }
- }};
-}
-
-/// Return an [`Error`](crate::Error) of a kind with a formatted message from
-/// the current function.
-///
-/// `bail!(kind, fmt, args...)` formats the message as `format!` does and
-/// records the source location in the backtrace.
-///
-/// # Arguments
-///
-/// * `kind` - The kind of the error, such as
[`VALUE_ERROR`](crate::VALUE_ERROR).
-/// * `fmt` - The format string of the message.
-/// * `args` - The format arguments.
-///
-/// # Returns
-///
-/// Returns `Err` of the error from the enclosing function.
-///
-/// # Examples
-///
-/// ```
-/// use tvm_ffi::{bail, Result, INDEX_ERROR};
-///
-/// fn get(items: &[i64], index: usize) -> Result<i64> {
-/// if index >= items.len() {
-/// bail!(INDEX_ERROR, "index {} out of range for length {}", index,
items.len());
-/// }
-/// Ok(items[index])
-/// }
-///
-/// assert_eq!(get(&[1, 2], 1)?, 2);
-/// assert_eq!(get(&[1, 2], 5).unwrap_err().message(), "index 5 out of range
for length 2");
-/// # Ok::<(), tvm_ffi::Error>(())
-/// ```
-#[macro_export]
-macro_rules! bail {
- ($error_kind:expr, $fmt:expr $(, $args:expr)* $(,)?) => {{
- let context = format!(
- " File \"{}\", line {}, in {}\n",
- file!(),
- line!(),
- $crate::function_name!()
- );
- return Err($crate::error::Error::new($error_kind, &format!($fmt $(,
$args)*), &context));
- }};
-}
-
-/// Return an [`Error`](crate::Error) from the current function unless a
-/// condition holds.
-///
-/// `ensure!(cond, kind, fmt, args...)` is [`bail!`](crate::bail)`(kind, fmt,
-/// args...)` when `cond` is false.
-///
-/// # Arguments
-///
-/// * `cond` - The condition that must hold.
-/// * `kind` - The kind of the error.
-/// * `fmt` - The format string of the message.
-/// * `args` - The format arguments.
-///
-/// # Returns
-///
-/// Returns `Err` of the error from the enclosing function when `cond` is
-/// false, and does nothing otherwise.
-///
-/// # Examples
-///
-/// ```
-/// use tvm_ffi::{ensure, Result, VALUE_ERROR};
-///
-/// fn sqrt(x: f64) -> Result<f64> {
-/// ensure!(x >= 0.0, VALUE_ERROR, "sqrt of negative number {}", x);
-/// Ok(x.sqrt())
-/// }
-///
-/// assert_eq!(sqrt(4.0)?, 2.0);
-/// assert!(sqrt(-1.0).is_err());
-/// # Ok::<(), tvm_ffi::Error>(())
-/// ```
-#[macro_export]
-macro_rules! ensure {
- ($cond:expr, $error_kind:expr, $fmt:expr $(, $args:expr)* $(,)?) => {{
- if !$cond {
- $crate::bail!($error_kind, $fmt $(, $args)*);
- }
- }};
-}
-
-/// Append the current source location to the backtrace of an `Err` result,
-/// and pass an `Ok` through.
-///
-/// # Arguments
-///
-/// * `result` - The result to attach the context to.
-///
-/// # Returns
-///
-/// The result, with the location appended to an error's backtrace.
-///
-/// # Examples
-///
-/// ```
-/// use tvm_ffi::{attach_context, bail, Result, RUNTIME_ERROR};
-///
-/// fn inner() -> Result<()> {
-/// bail!(RUNTIME_ERROR, "failed");
-/// }
-///
-/// fn outer() -> Result<()> {
-/// attach_context!(inner())
-/// }
-///
-/// assert!(outer().unwrap_err().backtrace().contains("outer"));
-/// ```
-#[macro_export]
-macro_rules! attach_context {
- ($error:expr) => {{
- match $error {
- Ok(value) => Ok(value),
- Err(error) => {
- let context = format!(
- " File \"{}\", line {}, in {}\n",
- file!(),
- line!(),
- $crate::function_name!()
- );
- Err($crate::error::Error::with_appended_backtrace(
- error, &context,
- ))
- }
- }
- }};
-}
-
// ----------------------------------------------------------------------------
// Macros for any definitions
// ----------------------------------------------------------------------------
diff --git a/rust/tvm-ffi/src/object.rs b/rust/tvm-ffi/src/object.rs
index 51ab2919..34236e7f 100644
--- a/rust/tvm-ffi/src/object.rs
+++ b/rust/tvm-ffi/src/object.rs
@@ -40,6 +40,28 @@ pub struct Object {
header: TVMFFIObject,
}
+impl Object {
+ #[allow(clippy::new_without_default)]
+ pub fn new() -> Self {
+ Self {
+ header: TVMFFIObject::new(),
+ }
+ }
+}
+
+unsafe impl ObjectCore for Object {
+ const TYPE_KEY: &'static str = "ffi.Object";
+ const TYPE_DEPTH: i32 = 0;
+ #[inline]
+ fn type_index() -> i32 {
+ TypeIndex::kTVMFFIStaticObjectBegin as i32
+ }
+ #[inline]
+ unsafe fn object_header_mut(this: &mut Self) -> &mut TVMFFIObject {
+ &mut this.header
+ }
+}
+
/// Arc-like wrapper for Object that allows shared ownership
///
/// \tparam T The type of the object to be wrapped
@@ -49,6 +71,213 @@ pub struct ObjectArc<T: ObjectCore> {
_phantom: std::marker::PhantomData<T>,
}
+impl ObjectArc<Object> {
+ /// Allocate a synchronous runtime payload that may contain borrowed Rust
state.
+ ///
+ /// # Safety
+ /// `T` begins with a valid Object header and is ABI-compatible with the
+ /// registered `type_index`. The caller must keep this owner scoped to the
+ /// payload's borrows and ensure every temporary strong and weak reference
+ /// is released before those borrows end. Neither native handles nor this
+ /// erased owner may escape that synchronous scope. Destruction uses T's
+ /// exact layout and drop function, not the erased Object layout.
+ /// The erased owner and handles must remain on this thread unless the
+ /// concrete payload permits sharing or transfer; erasure grants neither.
+ pub(crate) unsafe fn new_opaque_unchecked<T>(data: T, type_index: i32) ->
Self {
+ Self {
+ ptr: allocate_object(data, type_index).cast(),
+ _phantom: std::marker::PhantomData,
+ }
+ }
+}
+
+impl<T: ObjectCore> ObjectArc<T> {
+ /// Allocate an initialized object at its stable managed address.
+ pub fn new(data: T) -> Self {
+ // Resolve metadata before moving the payload into its allocation.
+ let type_index = T::type_index();
+ unsafe {
+ Self {
+ ptr: allocate_object(data, type_index),
+ _phantom: std::marker::PhantomData,
+ }
+ }
+ }
+
+ /// Allocate a stable object followed by its type's extra item storage.
+ pub fn new_with_extra_items<U>(data: T) -> Self
+ where
+ T: ObjectCoreWithExtraItems<ExtraItem = U>,
+ {
+ unsafe {
+ // ensure strict alignment requirements
+ // so we can have { T, U*extra_items } layout
+ assert_eq!(std::mem::align_of::<T>() % std::mem::align_of::<U>(),
0);
+ assert_eq!(std::mem::size_of::<T>() % std::mem::align_of::<U>(),
0);
+ let extra_items_count = T::extra_items_count(&data);
+ let layout = unsafe_::extra_items_layout::<T,
U>(extra_items_count);
+ let raw_data_ptr = std::alloc::alloc(layout);
+ if raw_data_ptr.is_null() {
+ std::alloc::handle_alloc_error(layout);
+ }
+ let ptr = raw_data_ptr as *mut T;
+ std::ptr::write(ptr, data);
+ // now override the header directly
+ std::ptr::write(
+ ptr as *mut TVMFFIObject,
+ TVMFFIObject {
+ combined_ref_count:
AtomicU64::new(COMBINED_REF_COUNT_BOTH_ONE),
+ type_index: T::type_index(),
+ __padding: 0,
+ deleter:
Some(unsafe_::object_deleter_for_new_with_extra_items::<T, U>),
+ },
+ );
+ // move into the object arc ptr
+ Self {
+ ptr: std::ptr::NonNull::new_unchecked(ptr),
+ _phantom: std::marker::PhantomData,
+ }
+ }
+ }
+
+ /// Whether `this` holds a null pointer, as all-zero bytes do: no
+ /// `ObjectArc`, but the field of an object that a creator left undefined.
+ ///
+ /// See also [`AnyCompatible::is_undefined`].
+ ///
+ /// # Safety
+ ///
+ /// `this` points to an `ObjectArc<T>` or to all-zero bytes.
+ #[inline]
+ pub unsafe fn is_null(this: *const Self) -> bool {
+ (*std::ptr::addr_of!((*this).ptr).cast::<*const T>()).is_null()
+ }
+
+ /// Move a previously allocated object into the ObjectArc
+ ///
+ /// # Arguments
+ /// * `ptr` - The raw pointer to move into the ObjectArc
+ ///
+ /// # Returns
+ /// * `ObjectArc<T>` - The ObjectArc
+ ///
+ /// # Safety
+ /// `ptr` must own one strong reference to a live, stable managed
allocation
+ /// whose dynamic type is compatible with `T` and whose payload is valid.
+ /// Ownership of that reference is transferred into the returned handle.
+ #[inline(always)]
+ pub unsafe fn from_raw(ptr: *const T) -> Self {
+ Self {
+ ptr: std::ptr::NonNull::new_unchecked(ptr as *mut T),
+ _phantom: std::marker::PhantomData,
+ }
+ }
+
+ /// Move the ObjectArc into a raw pointer
+ ///
+ /// # Arguments
+ /// * `this` - The ObjectArc to move into a raw pointer
+ ///
+ /// # Returns
+ /// * `*const T` - The raw pointer, which owns the strong reference; pass
it
+ /// back to [`ObjectArc::from_raw`] to release it, as std
`Arc::into_raw`.
+ #[inline(always)]
+ pub fn into_raw(this: Self) -> *const T {
+ let droped_this = std::mem::ManuallyDrop::new(this);
+ droped_this.ptr.as_ptr() as *const T
+ }
+
+ /// Get the raw pointer from the ObjectArc
+ ///
+ /// Caller should view this as a non-owning reference
+ ///
+ /// # Arguments
+ /// * `this` - The ObjectArc to get the raw pointer
+ ///
+ /// # Returns
+ /// * `*const T` - The raw pointer, as std `Arc::as_ptr`. Dereferencing it
+ /// must respect existing borrows, and it cannot outlive an owning
handle.
+ #[inline]
+ pub fn as_ptr(this: &Self) -> *const T {
+ this.ptr.as_ptr() as *const T
+ }
+
+ /// Get the raw mutable pointer from the ObjectArc
+ ///
+ /// Caller should view this as a non-owning reference
+ ///
+ /// # Arguments
+ /// * `this` - The ObjectArc to get the raw pointer
+ ///
+ /// # Returns
+ /// * `*mut T` - The raw pointer. The handle need not be unique: writing
+ /// through it needs exclusive access to the object, and must not move or
+ /// replace the object or its header.
+ #[inline]
+ pub fn as_mut_ptr(this: &mut Self) -> *mut T {
+ this.ptr.as_ptr()
+ }
+
+ /// Get the strong reference count of the ObjectArc
+ ///
+ /// # Arguments
+ /// * `this` - The ObjectArc to get the strong reference count
+ ///
+ /// # Returns
+ /// * `usize` - The strong reference count
+ #[inline]
+ pub fn strong_count(this: &Self) -> usize {
+ unsafe {
unsafe_::strong_count(this.ptr.as_ptr().cast::<TVMFFIObject>()) }
+ }
+
+ /// Whether this is the only reference, so it may be mutated in place.
+ #[inline]
+ pub(crate) fn is_unique(this: &Self) -> bool {
+ unsafe { unsafe_::is_unique(this.ptr.as_ptr().cast::<TVMFFIObject>()) }
+ }
+
+ /// Get the weak reference count of the ObjectArc
+ ///
+ /// # Arguments
+ /// * `this` - The ObjectArc to get the weak reference count
+ ///
+ /// # Returns
+ /// * `usize` - The weak reference count
+ #[inline]
+ pub fn weak_count(this: &Self) -> usize {
+ unsafe { unsafe_::weak_count(this.ptr.as_ptr().cast::<TVMFFIObject>())
}
+ }
+}
+
+// implement Deref for ObjectArc
+impl<T: ObjectCore> Deref for ObjectArc<T> {
+ type Target = T;
+ #[inline]
+ fn deref(&self) -> &Self::Target {
+ unsafe { self.ptr.as_ref() }
+ }
+}
+
+// implement Drop for ObjectArc
+impl<T: ObjectCore> Drop for ObjectArc<T> {
+ #[inline(always)]
+ fn drop(&mut self) {
+ unsafe { unsafe_::dec_ref(self.ptr.as_ptr().cast::<TVMFFIObject>()) }
+ }
+}
+
+// implement Clone for ObjectArc
+impl<T: ObjectCore> Clone for ObjectArc<T> {
+ #[inline]
+ fn clone(&self) -> Self {
+ unsafe { unsafe_::inc_ref(self.ptr.as_ptr().cast::<TVMFFIObject>()) }
+ Self {
+ ptr: self.ptr,
+ _phantom: std::marker::PhantomData,
+ }
+ }
+}
+
/// An object type whose shared instances are never mutated, so `ObjectArc<T>`
is
/// `Send + Sync` when `T`'s fields are.
///
@@ -65,6 +294,17 @@ unsafe impl<T: ImmutableObject + Send + Sync> Send for
ObjectArc<T> {}
// SAFETY: as for `Send`.
unsafe impl<T: ImmutableObject + Send + Sync> Sync for ObjectArc<T> {}
+/// A generic managed reference to a TVM-FFI object.
+///
+/// Holds an `ObjectArc<Object>` that manages the underlying object's lifetime
+/// through reference counting.
+#[repr(C)]
+#[derive(ObjectRef, Clone)]
+#[object_ref(skip_deref)]
+pub struct ObjectRef {
+ data: ObjectArc<Object>,
+}
+
/// The native representation of an object-system type.
///
/// # Safety
@@ -351,17 +591,6 @@ pub unsafe trait ObjectRefAnyCompatible: ObjectRefCore {
}
}
-/// Write the object cell of the live object `ptr`.
-#[inline(always)]
-unsafe fn write_object_cell<T: ObjectCore>(ptr: *const T, data: &mut
TVMFFIAny) {
- let object = ptr as *mut TVMFFIObject;
- data.type_index = (*object).type_index;
- // SAFETY: a live object's header holds its registered object-range type
index.
- std::hint::assert_unchecked(data.type_index >=
TypeIndex::kTVMFFIStaticObjectBegin as i32);
- data.small_str_len = 0;
- data.data_union.v_obj = object;
-}
-
// The object cell encoding, with the strict check `is_instance_of` then
// `extra_check`, and `convert_fallback` after it in converting casts.
unsafe impl<T: ObjectRefAnyCompatible> AnyCompatible for T {
@@ -603,11 +832,44 @@ pub fn is_instance_of<Target:
ObjectCore>(object_type_index: i32) -> bool {
}
}
-/// The strict downcast from the root [`ObjectRef`] to `B`, for the containers'
-/// `TryFrom<ObjectRef>`, which are hand-written as their upcasts are.
-#[inline(always)]
-pub(crate) fn root_try_into<B>(value: ObjectRef) -> crate::error::Result<B>
-where
+/// Format a managed reference as its runtime type key and object address, as
+/// the `Debug` implementation of `#[derive(ObjectRef)]` does.
+///
+/// Call it from `Debug` for a reference type implemented by hand, such as a
+/// typed view over another reference, so that `downcast(..).unwrap()` and
+/// `assert_eq!` can print it.
+///
+/// ```
+/// use tvm_ffi::object::{fmt_object_ref, ObjectRef};
+/// use tvm_ffi::Array;
+/// struct Shown(ObjectRef);
+/// impl std::fmt::Debug for Shown {
+/// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+/// fmt_object_ref(&self.0, f)
+/// }
+/// }
+/// let shown = Shown(Array::from_vec(vec![1_i64]).into());
+/// assert!(format!("{shown:?}").starts_with("ffi.Array@"));
+/// ```
+pub fn fmt_object_ref<T: ObjectRefCore>(
+ this: &T,
+ f: &mut std::fmt::Formatter<'_>,
+) -> std::fmt::Result {
+ let object = ObjectArc::as_ptr(T::data(this)).cast::<TVMFFIObject>();
+ // SAFETY: `this` keeps the object, and so its header, alive.
+ let type_index = unsafe { (*object).type_index };
+ write!(f, "{}@{:p}", type_key_of_index(type_index), object)
+}
+
+//
-----------------------------------------------------------------------------
+// Helpers
+//
-----------------------------------------------------------------------------
+
+/// The strict downcast from the root [`ObjectRef`] to `B`, for the containers'
+/// `TryFrom<ObjectRef>`, which are hand-written as their upcasts are.
+#[inline(always)]
+pub(crate) fn root_try_into<B>(value: ObjectRef) -> crate::error::Result<B>
+where
B: ObjectRefCore,
{
value
@@ -640,35 +902,6 @@ pub(crate) fn into_root_object_ref<S:
ObjectRefCore>(value: S) -> ObjectRef {
}
}
-/// Format a managed reference as its runtime type key and object address, as
-/// the `Debug` implementation of `#[derive(ObjectRef)]` does.
-///
-/// Call it from `Debug` for a reference type implemented by hand, such as a
-/// typed view over another reference, so that `downcast(..).unwrap()` and
-/// `assert_eq!` can print it.
-///
-/// ```
-/// use tvm_ffi::object::{fmt_object_ref, ObjectRef};
-/// use tvm_ffi::Array;
-/// struct Shown(ObjectRef);
-/// impl std::fmt::Debug for Shown {
-/// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-/// fmt_object_ref(&self.0, f)
-/// }
-/// }
-/// let shown = Shown(Array::from_vec(vec![1_i64]).into());
-/// assert!(format!("{shown:?}").starts_with("ffi.Array@"));
-/// ```
-pub fn fmt_object_ref<T: ObjectRefCore>(
- this: &T,
- f: &mut std::fmt::Formatter<'_>,
-) -> std::fmt::Result {
- let object = ObjectArc::as_ptr(T::data(this)).cast::<TVMFFIObject>();
- // SAFETY: `this` keeps the object, and so its header, alive.
- let type_index = unsafe { (*object).type_index };
- write!(f, "{}@{:p}", type_key_of_index(type_index), object)
-}
-
/// The registered type key of `type_index`, or `<type_index N>` when it has
none.
pub(crate) fn type_key_of_index(type_index: i32) -> String {
unsafe {
@@ -681,15 +914,37 @@ pub(crate) fn type_key_of_index(type_index: i32) ->
String {
}
}
-/// A generic managed reference to a TVM-FFI object.
-///
-/// Holds an `ObjectArc<Object>` that manages the underlying object's lifetime
-/// through reference counting.
-#[repr(C)]
-#[derive(ObjectRef, Clone)]
-#[object_ref(skip_deref)]
-pub struct ObjectRef {
- data: ObjectArc<Object>,
+/// Write the object cell of the live object `ptr`.
+#[inline(always)]
+unsafe fn write_object_cell<T: ObjectCore>(ptr: *const T, data: &mut
TVMFFIAny) {
+ let object = ptr as *mut TVMFFIObject;
+ data.type_index = (*object).type_index;
+ // SAFETY: a live object's header holds its registered object-range type
index.
+ std::hint::assert_unchecked(data.type_index >=
TypeIndex::kTVMFFIStaticObjectBegin as i32);
+ data.small_str_len = 0;
+ data.data_union.v_obj = object;
+}
+
+// Allocate and initialize the normal FFI header while preserving the concrete
+// payload's alignment and drop/layout deleter. The payload starts with an
Object.
+unsafe fn allocate_object<T>(data: T, type_index: i32) -> NonNull<T> {
+ let layout = std::alloc::Layout::new::<T>();
+ let allocation = std::alloc::alloc(layout);
+ if allocation.is_null() {
+ std::alloc::handle_alloc_error(layout);
+ }
+ let ptr = allocation.cast::<T>();
+ std::ptr::write(ptr, data);
+ std::ptr::write(
+ ptr.cast::<TVMFFIObject>(),
+ TVMFFIObject {
+ combined_ref_count: AtomicU64::new(COMBINED_REF_COUNT_BOTH_ONE),
+ type_index,
+ __padding: 0,
+ deleter: Some(unsafe_::object_deleter_for_new::<T>),
+ },
+ );
+ NonNull::new_unchecked(ptr)
}
/// Unsafe operations on object
@@ -828,254 +1083,3 @@ pub mod unsafe_ {
}
}
}
-
-impl Object {
- #[allow(clippy::new_without_default)]
- pub fn new() -> Self {
- Self {
- header: TVMFFIObject::new(),
- }
- }
-}
-
-unsafe impl ObjectCore for Object {
- const TYPE_KEY: &'static str = "ffi.Object";
- const TYPE_DEPTH: i32 = 0;
- #[inline]
- fn type_index() -> i32 {
- TypeIndex::kTVMFFIStaticObjectBegin as i32
- }
- #[inline]
- unsafe fn object_header_mut(this: &mut Self) -> &mut TVMFFIObject {
- &mut this.header
- }
-}
-
-// Allocate and initialize the normal FFI header while preserving the concrete
-// payload's alignment and drop/layout deleter. The payload starts with an
Object.
-unsafe fn allocate_object<T>(data: T, type_index: i32) -> NonNull<T> {
- let layout = std::alloc::Layout::new::<T>();
- let allocation = std::alloc::alloc(layout);
- if allocation.is_null() {
- std::alloc::handle_alloc_error(layout);
- }
- let ptr = allocation.cast::<T>();
- std::ptr::write(ptr, data);
- std::ptr::write(
- ptr.cast::<TVMFFIObject>(),
- TVMFFIObject {
- combined_ref_count: AtomicU64::new(COMBINED_REF_COUNT_BOTH_ONE),
- type_index,
- __padding: 0,
- deleter: Some(unsafe_::object_deleter_for_new::<T>),
- },
- );
- NonNull::new_unchecked(ptr)
-}
-
-impl ObjectArc<Object> {
- /// Allocate a synchronous runtime payload that may contain borrowed Rust
state.
- ///
- /// # Safety
- /// `T` begins with a valid Object header and is ABI-compatible with the
- /// registered `type_index`. The caller must keep this owner scoped to the
- /// payload's borrows and ensure every temporary strong and weak reference
- /// is released before those borrows end. Neither native handles nor this
- /// erased owner may escape that synchronous scope. Destruction uses T's
- /// exact layout and drop function, not the erased Object layout.
- /// The erased owner and handles must remain on this thread unless the
- /// concrete payload permits sharing or transfer; erasure grants neither.
- pub(crate) unsafe fn new_opaque_unchecked<T>(data: T, type_index: i32) ->
Self {
- Self {
- ptr: allocate_object(data, type_index).cast(),
- _phantom: std::marker::PhantomData,
- }
- }
-}
-
-impl<T: ObjectCore> ObjectArc<T> {
- /// Allocate an initialized object at its stable managed address.
- pub fn new(data: T) -> Self {
- // Resolve metadata before moving the payload into its allocation.
- let type_index = T::type_index();
- unsafe {
- Self {
- ptr: allocate_object(data, type_index),
- _phantom: std::marker::PhantomData,
- }
- }
- }
-
- /// Allocate a stable object followed by its type's extra item storage.
- pub fn new_with_extra_items<U>(data: T) -> Self
- where
- T: ObjectCoreWithExtraItems<ExtraItem = U>,
- {
- unsafe {
- // ensure strict alignment requirements
- // so we can have { T, U*extra_items } layout
- assert_eq!(std::mem::align_of::<T>() % std::mem::align_of::<U>(),
0);
- assert_eq!(std::mem::size_of::<T>() % std::mem::align_of::<U>(),
0);
- let extra_items_count = T::extra_items_count(&data);
- let layout = unsafe_::extra_items_layout::<T,
U>(extra_items_count);
- let raw_data_ptr = std::alloc::alloc(layout);
- if raw_data_ptr.is_null() {
- std::alloc::handle_alloc_error(layout);
- }
- let ptr = raw_data_ptr as *mut T;
- std::ptr::write(ptr, data);
- // now override the header directly
- std::ptr::write(
- ptr as *mut TVMFFIObject,
- TVMFFIObject {
- combined_ref_count:
AtomicU64::new(COMBINED_REF_COUNT_BOTH_ONE),
- type_index: T::type_index(),
- __padding: 0,
- deleter:
Some(unsafe_::object_deleter_for_new_with_extra_items::<T, U>),
- },
- );
- // move into the object arc ptr
- Self {
- ptr: std::ptr::NonNull::new_unchecked(ptr),
- _phantom: std::marker::PhantomData,
- }
- }
- }
-
- /// Whether `this` holds a null pointer, as all-zero bytes do: no
- /// `ObjectArc`, but the field of an object that a creator left undefined.
- ///
- /// See also [`AnyCompatible::is_undefined`].
- ///
- /// # Safety
- ///
- /// `this` points to an `ObjectArc<T>` or to all-zero bytes.
- #[inline]
- pub unsafe fn is_null(this: *const Self) -> bool {
- (*std::ptr::addr_of!((*this).ptr).cast::<*const T>()).is_null()
- }
-
- /// Move a previously allocated object into the ObjectArc
- ///
- /// # Arguments
- /// * `ptr` - The raw pointer to move into the ObjectArc
- ///
- /// # Returns
- /// * `ObjectArc<T>` - The ObjectArc
- ///
- /// # Safety
- /// `ptr` must own one strong reference to a live, stable managed
allocation
- /// whose dynamic type is compatible with `T` and whose payload is valid.
- /// Ownership of that reference is transferred into the returned handle.
- #[inline(always)]
- pub unsafe fn from_raw(ptr: *const T) -> Self {
- Self {
- ptr: std::ptr::NonNull::new_unchecked(ptr as *mut T),
- _phantom: std::marker::PhantomData,
- }
- }
-
- /// Move the ObjectArc into a raw pointer
- ///
- /// # Arguments
- /// * `this` - The ObjectArc to move into a raw pointer
- ///
- /// # Returns
- /// * `*const T` - The raw pointer, which owns the strong reference; pass
it
- /// back to [`ObjectArc::from_raw`] to release it, as std
`Arc::into_raw`.
- #[inline(always)]
- pub fn into_raw(this: Self) -> *const T {
- let droped_this = std::mem::ManuallyDrop::new(this);
- droped_this.ptr.as_ptr() as *const T
- }
-
- /// Get the raw pointer from the ObjectArc
- ///
- /// Caller should view this as a non-owning reference
- ///
- /// # Arguments
- /// * `this` - The ObjectArc to get the raw pointer
- ///
- /// # Returns
- /// * `*const T` - The raw pointer, as std `Arc::as_ptr`. Dereferencing it
- /// must respect existing borrows, and it cannot outlive an owning
handle.
- #[inline]
- pub fn as_ptr(this: &Self) -> *const T {
- this.ptr.as_ptr() as *const T
- }
-
- /// Get the raw mutable pointer from the ObjectArc
- ///
- /// Caller should view this as a non-owning reference
- ///
- /// # Arguments
- /// * `this` - The ObjectArc to get the raw pointer
- ///
- /// # Returns
- /// * `*mut T` - The raw pointer. The handle need not be unique: writing
- /// through it needs exclusive access to the object, and must not move or
- /// replace the object or its header.
- #[inline]
- pub fn as_mut_ptr(this: &mut Self) -> *mut T {
- this.ptr.as_ptr()
- }
-
- /// Get the strong reference count of the ObjectArc
- ///
- /// # Arguments
- /// * `this` - The ObjectArc to get the strong reference count
- ///
- /// # Returns
- /// * `usize` - The strong reference count
- #[inline]
- pub fn strong_count(this: &Self) -> usize {
- unsafe {
unsafe_::strong_count(this.ptr.as_ptr().cast::<TVMFFIObject>()) }
- }
-
- /// Whether this is the only reference, so it may be mutated in place.
- #[inline]
- pub(crate) fn is_unique(this: &Self) -> bool {
- unsafe { unsafe_::is_unique(this.ptr.as_ptr().cast::<TVMFFIObject>()) }
- }
-
- /// Get the weak reference count of the ObjectArc
- ///
- /// # Arguments
- /// * `this` - The ObjectArc to get the weak reference count
- ///
- /// # Returns
- /// * `usize` - The weak reference count
- #[inline]
- pub fn weak_count(this: &Self) -> usize {
- unsafe { unsafe_::weak_count(this.ptr.as_ptr().cast::<TVMFFIObject>())
}
- }
-}
-
-// implement Deref for ObjectArc
-impl<T: ObjectCore> Deref for ObjectArc<T> {
- type Target = T;
- #[inline]
- fn deref(&self) -> &Self::Target {
- unsafe { self.ptr.as_ref() }
- }
-}
-
-// implement Drop for ObjectArc
-impl<T: ObjectCore> Drop for ObjectArc<T> {
- #[inline(always)]
- fn drop(&mut self) {
- unsafe { unsafe_::dec_ref(self.ptr.as_ptr().cast::<TVMFFIObject>()) }
- }
-}
-
-// implement Clone for ObjectArc
-impl<T: ObjectCore> Clone for ObjectArc<T> {
- #[inline]
- fn clone(&self) -> Self {
- unsafe { unsafe_::inc_ref(self.ptr.as_ptr().cast::<TVMFFIObject>()) }
- Self {
- ptr: self.ptr,
- _phantom: std::marker::PhantomData,
- }
- }
-}
diff --git a/tests/python/test_stubgen_rust.py
b/tests/python/test_stubgen_rust.py
index d60e5708..445ba40a 100644
--- a/tests/python/test_stubgen_rust.py
+++ b/tests/python/test_stubgen_rust.py
@@ -1666,7 +1666,9 @@ def test_blocks_are_named_by_the_module_of_their_file(
text = nested.read_text(encoding="utf-8")
assert "use crate::TestCxxClassBase;\nuse crate::TestCxxClassBaseObj;\n"
in text
assert " base: TestCxxClassBaseObj," in text
- assert "tvm_ffi::unsafe_impl_object_ref_upcast!(TestCxxClassDerived =>
TestCxxClassBase);" in text
+ assert (
+ "tvm_ffi::unsafe_impl_object_ref_upcast!(TestCxxClassDerived =>
TestCxxClassBase);" in text
+ )
assert "super::" not in text
# Moving the parent's block moves the path, not the type key.
lib.write_text("", encoding="utf-8")
