This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm-ffi.git
The following commit(s) were added to refs/heads/main by this push:
new e6d897d1 [REFACTOR][Rust] Clarify structural traversal utilities and
initialization (#906)
e6d897d1 is described below
commit e6d897d172b450502668291032af33f34f6e20f4
Author: Tianqi Chen <[email protected]>
AuthorDate: Tue Nov 3 21:46:04 2026 -0500
[REFACTOR][Rust] Clarify structural traversal utilities and initialization
(#906)
Add a public `ColdInitLazyLock` that shares the initialized check and
outlined initialization path. Use it for structural traversal columns,
runtime type indices and cached global functions, keeping Once setup
outside hot callers where the compiler preserves the outline boundary.
Keep outlined errors alongside their traversals and share flat
raw-carrier comparison and result-conversion utilities. Fold the
single-use ownership fast path into its conversion caller, use direct
carrier access for trivial conversions, and document ownership, native
ABI transport and performance-sensitive implementation choices.
---
rust/tvm-ffi-macros/src/object_macros.rs | 30 +-
rust/tvm-ffi/src/extra/cold_init_lazy_lock.rs | 77 +++++
rust/tvm-ffi/src/extra/mod.rs | 1 +
rust/tvm-ffi/src/extra/structural_common.rs | 111 +++++---
rust/tvm-ffi/src/extra/structural_map.rs | 21 +-
rust/tvm-ffi/src/extra/structural_mutate.rs | 387 ++++++++++++++------------
rust/tvm-ffi/src/extra/structural_visit.rs | 199 +++++++------
rust/tvm-ffi/src/extra/structural_walk.rs | 4 +-
rust/tvm-ffi/src/extra/unchanged.rs | 4 +-
rust/tvm-ffi/src/macros.rs | 13 +-
10 files changed, 503 insertions(+), 344 deletions(-)
diff --git a/rust/tvm-ffi-macros/src/object_macros.rs
b/rust/tvm-ffi-macros/src/object_macros.rs
index fc9b2449..35cefce0 100644
--- a/rust/tvm-ffi-macros/src/object_macros.rs
+++ b/rust/tvm-ffi-macros/src/object_macros.rs
@@ -136,7 +136,7 @@ pub(crate) fn expand_object(
quote! {
#[inline]
fn type_index() -> i32 {
- static TYPE_INDEX: std::sync::LazyLock<i32> =
std::sync::LazyLock::new(||
+ static TYPE_INDEX:
#tvm_ffi_crate::extra::cold_init_lazy_lock::ColdInitLazyLock<i32> =
#tvm_ffi_crate::extra::cold_init_lazy_lock::ColdInitLazyLock::new(||
unsafe {
// The parent first, since registering it takes the
// same lock.
@@ -165,18 +165,7 @@ pub(crate) fn expand_object(
tindex
}
);
- // Keep Once's initialization descriptors in the cold
helper,
- // rather than in every inlined runtime type-index caller.
- #[cold]
- #[inline(never)]
- fn initialize_type_index() -> i32 {
- *std::sync::LazyLock::force(&TYPE_INDEX)
- }
- if let Some(index) = std::sync::LazyLock::get(&TYPE_INDEX)
{
- *index
- } else {
- initialize_type_index()
- }
+ *TYPE_INDEX
}
}
}
@@ -184,7 +173,7 @@ pub(crate) fn expand_object(
quote! {
#[inline]
fn type_index() -> i32 {
- static TYPE_INDEX: std::sync::LazyLock<i32> =
std::sync::LazyLock::new(||
+ static TYPE_INDEX:
#tvm_ffi_crate::extra::cold_init_lazy_lock::ColdInitLazyLock<i32> =
#tvm_ffi_crate::extra::cold_init_lazy_lock::ColdInitLazyLock::new(||
unsafe {
let type_key_arg =
#tvm_ffi_crate::tvm_ffi_sys::TVMFFIByteArray::from_str(#type_key);
@@ -202,18 +191,7 @@ pub(crate) fn expand_object(
tindex
}
);
- // Keep Once's initialization descriptors in the cold
helper,
- // rather than in every inlined runtime type-index caller.
- #[cold]
- #[inline(never)]
- fn initialize_type_index() -> i32 {
- *std::sync::LazyLock::force(&TYPE_INDEX)
- }
- if let Some(index) = std::sync::LazyLock::get(&TYPE_INDEX)
{
- *index
- } else {
- initialize_type_index()
- }
+ *TYPE_INDEX
}
}
}
diff --git a/rust/tvm-ffi/src/extra/cold_init_lazy_lock.rs
b/rust/tvm-ffi/src/extra/cold_init_lazy_lock.rs
new file mode 100644
index 00000000..b53d8e2a
--- /dev/null
+++ b/rust/tvm-ffi/src/extra/cold_init_lazy_lock.rs
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+//! Lazy initialization with an outlined first-access path.
+
+use std::ops::Deref;
+use std::sync::LazyLock;
+
+/// A lazy value that keeps initialization in a separate cold function.
+///
+/// Construct it with an initializer and dereference it to access the value.
+/// Initialization runs once; concurrent callers wait, and an initializer panic
+/// poisons the value, following [`LazyLock`].
+///
+/// # Example
+///
+/// ```
+/// use tvm_ffi::extra::cold_init_lazy_lock::ColdInitLazyLock;
+///
+/// static NAME: ColdInitLazyLock<String> = ColdInitLazyLock::new(||
"ready".into());
+/// assert_eq!(NAME.as_str(), "ready");
+/// ```
+///
+/// # Why avoid inlining initialization?
+///
+/// Inlining initialization can enlarge the caller's stack frame even when the
+/// value is already initialized. We keep initialization in a separate cold
+/// function and inline only the initialized check.
+pub struct ColdInitLazyLock<T, F = fn() -> T> {
+ inner: LazyLock<T, F>,
+}
+
+impl<T, F: FnOnce() -> T> ColdInitLazyLock<T, F> {
+ /// Store `initializer` for execution on the first dereference.
+ ///
+ /// Construction does not execute the initializer or allocate wrapper
storage.
+ pub const fn new(initializer: F) -> Self {
+ Self {
+ inner: LazyLock::new(initializer),
+ }
+ }
+}
+
+impl<T, F: FnOnce() -> T> Deref for ColdInitLazyLock<T, F> {
+ type Target = T;
+
+ #[inline(always)]
+ fn deref(&self) -> &T {
+ if let Some(value) = LazyLock::get(&self.inner) {
+ value
+ } else {
+ force_outlined(&self.inner)
+ }
+ }
+}
+
+// Keep the lock's first-access machinery outside initialized callers' frames.
+#[cold]
+#[inline(never)]
+fn force_outlined<T, F: FnOnce() -> T>(lock: &LazyLock<T, F>) -> &T {
+ LazyLock::force(lock)
+}
diff --git a/rust/tvm-ffi/src/extra/mod.rs b/rust/tvm-ffi/src/extra/mod.rs
index a622c954..911e9f09 100644
--- a/rust/tvm-ffi/src/extra/mod.rs
+++ b/rust/tvm-ffi/src/extra/mod.rs
@@ -16,6 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+pub mod cold_init_lazy_lock;
pub mod dispatch;
pub mod marker;
pub mod module;
diff --git a/rust/tvm-ffi/src/extra/structural_common.rs
b/rust/tvm-ffi/src/extra/structural_common.rs
index 87d4f109..1ebc228b 100644
--- a/rust/tvm-ffi/src/extra/structural_common.rs
+++ b/rust/tvm-ffi/src/extra/structural_common.rs
@@ -18,7 +18,7 @@
*/
use crate::any::{Any, AnyView};
-use crate::error::Error;
+use crate::error::{Error, Result, RUNTIME_ERROR};
use crate::function::Function;
use crate::object::{self, ObjectRefCore};
use crate::reflection::FieldGetter;
@@ -27,11 +27,43 @@ use crate::tvm_ffi_sys::{
TVMFFITypeKeyToIndex,
};
+//
-----------------------------------------------------------------------------
+// Shared outlined errors
+//
-----------------------------------------------------------------------------
+// Perf note: Keep construction, formatting and checked-conversion scratch
inside
+// these cold outline boundaries, not in successful traversal callers' frames.
+
+#[cold]
+#[inline(never)]
+pub(super) fn runtime_error(message: &str) -> Error {
+ Error::new(RUNTIME_ERROR, message, "")
+}
+
+#[cold]
+#[inline(never)]
+pub(super) fn missing_field_getter_error(operation: &str, name: &str) -> Error
{
+ runtime_error(&format!(
+ "native {operation}: reflected field `{name}` has no getter"
+ ))
+}
+
+#[cold]
+#[inline(never)]
+pub(super) fn with_structural_field_error_context(
+ error: Error,
+ operation: &str,
+ name: &str,
+) -> Error {
+ with_structural_error_context(error, operation, &format!("field `{name}`"))
+}
+
/// Consume an owning error carrier while keeping checked-conversion scratch
/// inside the cold path instead of successful recursive traversal frames.
///
/// # Safety
/// The fields must describe one owning TVMFFIAny that has not been consumed.
+// Perf note: Construct the addressable Any and perform checked Error
conversion
+// inside this outline boundary, not in the successful recursive caller's
frame.
#[cold]
#[inline(never)]
pub(super) unsafe fn structural_error_from_raw(
@@ -67,6 +99,9 @@ pub(crate) fn type_key_of(type_index: i32) -> String {
}
}
+// Perf note: cold marks these error-composition paths as uncommon; inlining
+// within error paths is compiler-selected. Use explicit inline(never) at
boundaries
+// where scratch must stay outside recursive callers.
/// Add one structural traversal frame to an error's backtrace.
#[cold]
pub(crate) fn with_structural_error_context(error: Error, operation: &str,
frame: &str) -> Error {
@@ -87,7 +122,9 @@ pub(crate) fn with_visit_error_context(error: Error, raw:
TVMFFIAny) -> Error {
))
.ok()?;
}
- let node = unsafe { borrowed_view(&raw)
}.try_as::<object::ObjectRef>()?;
+ // SAFETY: the traversal input remains live during annotation.
Conversion
+ // acquires ownership before the temporary view ends; raw is an object
tag.
+ let node = unsafe { AnyView::from_raw_ffi_any(raw)
}.try_as::<object::ObjectRef>()?;
let mut previous = error.extra_context();
let mut nodes = Vec::new();
if let Some(prior) = previous.as_ref() {
@@ -189,48 +226,54 @@ macro_rules! impl_callback_chain_tuple_arities {
pub(crate) use impl_callback_chain_tuple_arities;
-/// Copy a borrowed FFI value when its owning representation is unchanged.
-///
-/// Raw string/byte views and `ObjectRValueRef` return `None` because they need
-/// the runtime's normalization or move logic. A null object pointer also
-/// returns `None` instead of constructing an invalid owning value.
-#[inline(always)]
-pub(crate) fn try_to_owned_without_normalization(raw: TVMFFIAny) ->
Option<Any> {
- if is_plain_inline(raw.type_index) {
- return Some(unsafe { Any::from_raw_ffi_any(raw) });
- }
- if raw.type_index >= TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
- let object = unsafe { raw.data_union.v_obj };
- if object.is_null() {
- return None;
- }
- unsafe { object::unsafe_::inc_ref(object) };
- return Some(unsafe { Any::from_raw_ffi_any(raw) });
- }
- None
-}
-
-pub(crate) use crate::any::is_plain_inline;
+//
-----------------------------------------------------------------------------
+// Raw carrier utilities
+//
-----------------------------------------------------------------------------
+/// Compare tags, inline metadata and payload bits without deep equality.
+///
+/// Object-backed values compare pointer identity. Both carriers must contain
+/// initialized payload bits; no pointee is accessed or ownership transferred.
#[inline(always)]
-pub(crate) fn same_shallow(lhs: TVMFFIAny, rhs: TVMFFIAny) -> bool {
+pub(crate) fn raw_tvm_ffi_any_same_as(lhs: TVMFFIAny, rhs: TVMFFIAny) -> bool {
lhs.type_index == rhs.type_index
&& lhs.small_str_len == rhs.small_str_len
&& unsafe { lhs.data_union.v_uint64 == rhs.data_union.v_uint64 }
}
-/// Borrow a traversal ABI cell for no longer than the cell's scope.
+/// Transfer an owning success or Error into exactly one raw result token.
///
-/// # Safety
-/// The cell must satisfy `AnyView::from_raw_ffi_any`'s requirements, including
-/// its unique, at-most-once conversion rule for rvalue slots. The traversal
-/// driver must keep the owning root, field, or replacement alive for the
borrow.
+/// This is the structural hook protocol: an Error tag represents failure, so
+/// decoding `Ok(Any::from(error))` also yields `Err(error)`. Callers that
carry
+/// Error objects as ordinary values must keep a separate result channel.
+/// No payload is cloned, normalized, or released by this transfer.
#[inline(always)]
-pub(crate) unsafe fn borrowed_view(raw: &TVMFFIAny) -> AnyView<'_> {
- AnyView::from_raw_ffi_any(*raw)
+pub(crate) fn result_into_raw_tvm_ffi_any(result: Result<Any>) -> TVMFFIAny {
+ unsafe {
+ match result {
+ Ok(value) => Any::into_raw_ffi_any(value),
+ Err(error) => Any::into_raw_ffi_any(Any::from(error)),
+ }
+ }
}
+/// Take ownership of one value returned by a structural traversal ABI hook.
+///
+/// # Safety
+///
+/// `raw` must contain one owning TVMFFIAny result that has not already been
+/// consumed. An Error object is converted into the Rust error channel, even
+/// if it was encoded from `Ok`; all other tags become an owning success.
+/// Borrowed views and rvalue slots must be normalized before this transfer.
#[inline(always)]
-pub(crate) fn raw_of(value: AnyView<'_>) -> TVMFFIAny {
- *value.as_raw_ffi_any()
+pub(crate) unsafe fn raw_tvm_ffi_any_into_result(raw: TVMFFIAny) ->
Result<Any> {
+ if raw.type_index != TVMFFITypeIndex::kTVMFFIError as i32 {
+ return Ok(Any::from_raw_ffi_any(raw));
+ }
+ core::hint::cold_path();
+ Err(structural_error_from_raw(
+ raw.type_index,
+ raw.small_str_len,
+ raw.data_union,
+ ))
}
diff --git a/rust/tvm-ffi/src/extra/structural_map.rs
b/rust/tvm-ffi/src/extra/structural_map.rs
index 9c0839dd..98a3bf9e 100644
--- a/rust/tvm-ffi/src/extra/structural_map.rs
+++ b/rust/tvm-ffi/src/extra/structural_map.rs
@@ -19,9 +19,7 @@
//! Ordered replacement callbacks and adapters for structural mapping.
-use super::structural_common::{
- borrowed_view, impl_callback_chain_tuple_arities, raw_of, same_shallow,
-};
+use super::structural_common::{impl_callback_chain_tuple_arities,
raw_tvm_ffi_any_same_as};
use super::structural_mutate::{
with_mutation_error_context, IntoMutateResult, MutationView,
StructuralMutateEngine,
StructuralMutatorView,
@@ -483,19 +481,21 @@ where
mutator: &mut StructuralMutatorView<'_, Self>,
) -> Result<Any> {
let mode = value.inplace_mode();
- let raw = raw_of(value.as_any_view());
+ let raw = *value.as_any_view().as_raw_ffi_any();
let kind = mutator.def_region_kind();
if PRE_ORDER {
// Snapshot permission before a callback can retain or release
ownership.
+ // SAFETY: the input owner remains live through this synchronous
callback;
+ // its higher-ranked view cannot escape into the owning result.
let Some(callback_result) = self
.dispatch
- .dispatch_tuple((unsafe { borrowed_view(&raw) }, kind))
+ .dispatch_tuple((unsafe { AnyView::from_raw_ffi_any(raw) },
kind))
else {
return mutator.default_mutate(self, value).map(Into::into);
};
let mapped =
callback_result.map_err(|error|
with_mutation_error_context(error, raw))?;
- if is_unchanged(&mapped) || same_shallow(raw,
*mapped.as_raw_ffi_any()) {
+ if is_unchanged(&mapped) || raw_tvm_ffi_any_same_as(raw,
*mapped.as_raw_ffi_any()) {
// Drop temporary ownership before default descent checks
uniqueness.
drop(mapped);
mutator.default_mutate(self, value).map(Into::into)
@@ -511,15 +511,16 @@ where
} else {
let mapped: Any = mutator.default_mutate(self, value)?.into();
let current = if is_unchanged(&mapped) {
- unsafe { borrowed_view(&raw) }
+ // SAFETY: Unchanged leaves the original input owner in place.
Borrow
+ // it only through the callback; the other branch borrows
mapped.
+ unsafe { AnyView::from_raw_ffi_any(raw) }
} else {
AnyView::from(&mapped)
};
match self.dispatch.dispatch_tuple((current, kind)) {
Some(Ok(result)) if is_unchanged(&result) => Ok(mapped),
- Some(result) => {
- result.map_err(|error| with_mutation_error_context(error,
raw_of(current)))
- }
+ Some(result) => result
+ .map_err(|error| with_mutation_error_context(error,
*current.as_raw_ffi_any())),
None => Ok(mapped),
}
}
diff --git a/rust/tvm-ffi/src/extra/structural_mutate.rs
b/rust/tvm-ffi/src/extra/structural_mutate.rs
index 8cf5de2a..2a6aa674 100644
--- a/rust/tvm-ffi/src/extra/structural_mutate.rs
+++ b/rust/tvm-ffi/src/extra/structural_mutate.rs
@@ -22,22 +22,43 @@
//! [`structural_mutate`] lets a mutator drive recursion, while
//! [`structural_map`] applies callbacks around engine-owned recursion.
+//
-----------------------------------------------------------------------------
+// Implementation overview
+//
-----------------------------------------------------------------------------
+//
+// This file is organized into:
+// - Data structures and mutation capabilities.
+// - Outlined errors and private utility functions.
+// - Engine implementation and the public entry point.
+//
+// We use narrowly scoped unsafe code for native interoperation and efficient
+// traversal. Safety and ownership contracts are documented at each boundary.
+//
+// The main performance strategies are:
+// - Outline cold work to keep its temporary storage out of recursive frames.
+// - Use _raw helpers and private extern "C-unwind" boundaries for register
+// transport on AArch64 while preserving Rust unwinding.
+// - Inline small conversions and forwarding code so ownership transfers and
+// redundant cleanup can optimize away.
+//
+// Local Perf note comments explain individual implementation choices.
+
use std::cell::RefCell;
use std::collections::{hash_map::Entry, HashMap};
use std::ffi::c_void;
use std::marker::PhantomData;
+use super::cold_init_lazy_lock::ColdInitLazyLock;
use super::marker::{Borrowed, Owned};
use std::ops::ControlFlow;
use std::ptr::NonNull;
use std::rc::Rc;
-use std::sync::LazyLock;
pub use super::structural_map::*;
use super::dispatch::Dispatcher;
-use crate::any::{Any, AnyView};
-use crate::error::{Error, Result, RUNTIME_ERROR, TYPE_ERROR};
+use crate::any::{is_plain_inline, Any, AnyView};
+use crate::error::{Error, Result, TYPE_ERROR};
use crate::function::Function;
use crate::object::{self, Object, ObjectArc, ObjectCore};
use crate::reflection::TypeAttrColumn;
@@ -52,9 +73,9 @@ use crate::tvm_ffi_sys::{
use crate::tvm_ffi_sys::{TVMFFIObjectHandle, TVMFFISEqHashKind};
use super::structural_common::{
- borrowed_view, raw_of, same_shallow, structural_error_from_raw,
- try_to_owned_without_normalization, type_key_of,
with_structural_error_context,
- with_visit_error_context,
+ missing_field_getter_error, raw_tvm_ffi_any_into_result,
raw_tvm_ffi_any_same_as,
+ result_into_raw_tvm_ffi_any, runtime_error, type_key_of,
with_structural_error_context,
+ with_structural_field_error_context, with_visit_error_context,
};
use super::structural_visit::{
field_def_region, for_each_field_info, free_var_child_region,
DefRegionKind,
@@ -110,7 +131,7 @@ const _: () = {
impl StructuralMutatorObj {
const TYPE_KEY: &'static str = "ffi.StructuralMutator";
fn type_index() -> i32 {
- static TYPE_INDEX: LazyLock<i32> = LazyLock::new(|| unsafe {
+ static TYPE_INDEX: ColdInitLazyLock<i32> = ColdInitLazyLock::new(||
unsafe {
let key =
TVMFFIByteArray::from_str(StructuralMutatorObj::TYPE_KEY);
let mut type_index = 0;
if TVMFFITypeKeyToIndex(&key, &mut type_index) != 0 {
@@ -149,6 +170,9 @@ impl Default for StructuralMutatorObj {
/// address reuse until removal. Values are owning replacements, including
/// [`Unchanged`] markers. Replacing a value does not acquire another key
reference;
/// lookup clones the replacement, and clearing or dropping releases every
owner.
+// Perf note: Store only the replacement Any; raw identity keys avoid a second
+// Any payload, but still own one retained reference per distinct key.
Replacing
+// a value does not retain the key again; clear/Drop release each key exactly
once.
#[derive(Default)]
pub struct StructuralVarRemap {
entries: HashMap<NonNull<TVMFFIObject>, Any>,
@@ -157,7 +181,7 @@ pub struct StructuralVarRemap {
impl StructuralVarRemap {
/// Look up an identity replacement previously stored for `var`.
pub fn get(&self, var: AnyView<'_>) -> Result<Option<Any>> {
- let key = object_identity_key(raw_of(var))?;
+ let key = object_identity_key(*var.as_raw_ffi_any())?;
Ok(self.entries.get(&key).cloned())
}
@@ -166,7 +190,7 @@ impl StructuralVarRemap {
/// A new key acquires one strong reference; replacing its value preserves
/// that reference. The replacement is cloned before modifying the table.
pub fn set(&mut self, var: AnyView<'_>, mutated_value: &Any) -> Result<()>
{
- let key = object_identity_key(raw_of(var))?;
+ let key = object_identity_key(*var.as_raw_ffi_any())?;
let value = mutated_value.clone();
match self.entries.entry(key) {
Entry::Occupied(mut entry) => {
@@ -200,6 +224,8 @@ impl StructuralVarRemap {
}
}
+// Perf note: Terminal destruction needs no replacement table or reusable
capacity.
+// Keep it separate from clear(), which leaves a valid remap ready for later
use.
impl Drop for StructuralVarRemap {
fn drop(&mut self) {
// Release keys first: their C ABI deleters cannot unwind. Field Drop
@@ -354,7 +380,7 @@ impl<'a> MutationView<'a> {
#[inline(always)]
pub(super) fn new(value: AnyView<'a>, mode: InplaceMode) -> Self {
Self {
- value: raw_of(value),
+ value: *value.as_raw_ffi_any(),
mode,
_scope: PhantomData,
_type: PhantomData,
@@ -400,7 +426,7 @@ impl<'a, T> MutationView<'a, T> {
pub fn as_any_view(&self) -> AnyView<'_> {
// SAFETY: the input owner keeps this payload live while the
capability is
// borrowed. The returned lifetime is bounded by self, not by 'a.
- unsafe { borrowed_view(&self.value) }
+ unsafe { AnyView::from_raw_ffi_any(self.value) }
}
/// Borrow an object or subtype as `U` without changing ownership or
permission.
@@ -577,7 +603,7 @@ impl<D> StructuralMutatorView<'_, D> {
let value = value.into();
unsafe {
let context =
StructuralMutateEngine::checked_dispatcher(self.runtime, dispatch);
- result_from_raw(StructuralMutateEngine::default_mutate_raw(
+
raw_tvm_ffi_any_into_result(StructuralMutateEngine::default_mutate_raw(
self.runtime,
context,
value.value,
@@ -591,7 +617,7 @@ impl<D> StructuralMutatorView<'_, D> {
pub fn var_remap_get(&mut self, dispatch: &mut D, var: AnyView<'_>) ->
Result<Option<Any>> {
unsafe {
StructuralMutateEngine::checked_dispatcher(self.runtime, dispatch);
- StructuralMutateEngine::runtime_var_remap_get(self.runtime,
raw_of(var))
+ StructuralMutateEngine::runtime_var_remap_get(self.runtime,
*var.as_raw_ffi_any())
}
}
@@ -599,7 +625,11 @@ impl<D> StructuralMutatorView<'_, D> {
pub fn var_remap_set(&mut self, dispatch: &mut D, var: AnyView<'_>, value:
&Any) -> Result<()> {
unsafe {
StructuralMutateEngine::checked_dispatcher(self.runtime, dispatch);
- StructuralMutateEngine::runtime_var_remap_set(self.runtime,
raw_of(var), value)
+ StructuralMutateEngine::runtime_var_remap_set(
+ self.runtime,
+ *var.as_raw_ffi_any(),
+ value,
+ )
}
}
}
@@ -633,8 +663,88 @@ where
}
//
-----------------------------------------------------------------------------
-// Private utility functions
+// Outlined errors
//
-----------------------------------------------------------------------------
+// Keep construction and formatting scratch outside successful traversal
frames.
+
+#[cold]
+#[inline(never)]
+fn missing_shallow_copy_error(type_index: i32, attribute: &str) -> Error {
+ Error::new(
+ TYPE_ERROR,
+ &format!(
+ "type `{}` cannot use reflected structural mutation because it
does not define `{attribute}`",
+ type_key_of(type_index)
+ ),
+ "",
+ )
+}
+
+#[cold]
+#[inline(never)]
+fn invalid_shallow_copy_hook_error(attribute: &str) -> Error {
+ Error::new(
+ TYPE_ERROR,
+ &format!("{attribute} must be an ffi.Function"),
+ "",
+ )
+}
+
+#[cold]
+#[inline(never)]
+fn invalid_shallow_copy_result_error() -> Error {
+ Error::new(
+ TYPE_ERROR,
+ "shallow copy callback must return a distinct object with the same
type as its input",
+ "",
+ )
+}
+
+#[cold]
+#[inline(never)]
+fn missing_field_setter_error(name: &str) -> Error {
+ Error::new(
+ TYPE_ERROR,
+ &format!("cannot structurally mutate field `{name}` because it does
not define a setter"),
+ "",
+ )
+}
+
+#[cold]
+#[inline(never)]
+fn invalid_field_offset_error(name: &str) -> Error {
+ runtime_error(&format!(
+ "native structural map: reflected field `{name}` has an invalid offset"
+ ))
+}
+
+#[cold]
+#[inline(never)]
+fn invalid_mutation_hook_error() -> Error {
+ Error::new(
+ TYPE_ERROR,
+ "__s_mutate__ must be an opaque function pointer or ffi.Function",
+ "",
+ )
+}
+
+#[cold]
+#[inline(never)]
+fn var_remap_key_error() -> Error {
+ Error::new(
+ TYPE_ERROR,
+ "variable-remap keys must be object-backed values",
+ "",
+ )
+}
+
+#[cold]
+#[inline(never)]
+fn unregistered_type_error(type_index: i32) -> Error {
+ runtime_error(&format!(
+ "native structural map: unregistered type index {type_index}"
+ ))
+}
// Keep panic construction out of the successful dispatcher identity check.
#[cold]
@@ -643,52 +753,32 @@ fn wrong_mutation_dispatcher() -> ! {
panic!("mutation context used with a different dispatcher instance")
}
-#[inline(always)]
-fn result_into_raw(result: Result<Any>) -> TVMFFIAny {
- unsafe {
- match result {
- Ok(value) => Any::into_raw_ffi_any(value),
- Err(error) => Any::into_raw_ffi_any(Any::from(error)),
- }
+#[cold]
+pub(super) fn with_mutation_error_context(error: Error, raw: TVMFFIAny) ->
Error {
+ if raw.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
+ error
+ } else {
+ with_structural_error_context(
+ with_visit_error_context(error, raw),
+ "map",
+ &format!("object `{}`", type_key_of(raw.type_index)),
+ )
}
}
-/// Take ownership of one value returned by a structural-mutation ABI hook.
-///
-/// # Safety
-///
-/// `raw` must contain one owning TVMFFIAny result that has not already been
-/// consumed. An Error object is converted into the Rust error channel.
-#[inline(always)]
-unsafe fn result_from_raw(raw: TVMFFIAny) -> Result<Any> {
- if raw.type_index != TVMFFITypeIndex::kTVMFFIError as i32 {
- return Ok(Any::from_raw_ffi_any(raw));
- }
- core::hint::cold_path();
- Err(structural_error_from_raw(
- raw.type_index,
- raw.small_str_len,
- raw.data_union,
- ))
-}
+//
-----------------------------------------------------------------------------
+// Private utility functions
+//
-----------------------------------------------------------------------------
fn shallow_copy(raw: TVMFFIAny) -> Result<Any> {
let Some(attr) = SHALLOW_COPY_COLUMN.get_raw(raw.type_index) else {
- return Err(Error::new(
- TYPE_ERROR,
- &format!(
- "type `{}` cannot use reflected structural mutation because it
does not define `{SHALLOW_COPY_ATTR}`",
- type_key_of(raw.type_index)
- ),
- "",
+ return Err(missing_shallow_copy_error(
+ raw.type_index,
+ SHALLOW_COPY_ATTR,
));
};
if attr.type_index != TVMFFITypeIndex::kTVMFFIFunction as i32 {
- return Err(Error::new(
- TYPE_ERROR,
- &format!("{SHALLOW_COPY_ATTR} must be an ffi.Function"),
- "",
- ));
+ return Err(invalid_shallow_copy_hook_error(SHALLOW_COPY_ATTR));
}
let function = unsafe { attr.data_union.v_obj };
if function.is_null() {
@@ -709,11 +799,7 @@ fn shallow_copy(raw: TVMFFIAny) -> Result<Any> {
|| result_pointer.is_null()
|| result_pointer == source_pointer
{
- return Err(Error::new(
- TYPE_ERROR,
- "shallow copy callback must return a distinct object with the same
type as its input",
- "",
- ));
+ return Err(invalid_shallow_copy_result_error());
}
Ok(result)
}
@@ -724,14 +810,7 @@ fn call_field_setter(
value: &TVMFFIAny,
) -> Result<()> {
if field.setter.is_null() {
- return Err(Error::new(
- TYPE_ERROR,
- &format!(
- "cannot structurally mutate field `{}` because it does not
define a setter",
- field.name.as_str()
- ),
- "",
- ));
+ return Err(missing_field_setter_error(field.name.as_str()));
}
let return_code = unsafe {
if field.flags & FLAG_SETTER_IS_FUNCTION == 0 {
@@ -773,15 +852,6 @@ fn object_identity_key(raw: TVMFFIAny) ->
Result<NonNull<TVMFFIObject>> {
})
}
-#[cold]
-fn var_remap_key_error() -> Error {
- Error::new(
- TYPE_ERROR,
- "variable-remap keys must be object-backed values",
- "",
- )
-}
-
#[inline]
fn checked_type_info(type_index: i32) -> Result<*const
crate::tvm_ffi_sys::TVMFFITypeInfo> {
let info = unsafe { TVMFFIGetTypeInfo(type_index) };
@@ -792,13 +862,6 @@ fn checked_type_info(type_index: i32) -> Result<*const
crate::tvm_ffi_sys::TVMFF
}
}
-#[cold]
-fn unregistered_type_error(type_index: i32) -> Error {
- runtime_error(&format!(
- "native structural map: unregistered type index {type_index}"
- ))
-}
-
#[inline(always)]
fn object_is_unique(raw: TVMFFIAny) -> bool {
if raw.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
@@ -808,15 +871,28 @@ fn object_is_unique(raw: TVMFFIAny) -> bool {
!pointer.is_null() && unsafe { object::unsafe_::strong_count(pointer) == 1
}
}
+/// Acquire an owning value from a valid borrowed carrier.
+///
+/// Inline values are copied and objects are retained once, including Error
objects
+/// as ordinary values. String/byte views are normalized by the runtime; an
rvalue
+/// slot transfers its reference. Internal callers keep borrowed resources
live and
+/// uniquely own each writable rvalue slot, converting it at most once. A null
+/// object pointer or failed runtime conversion returns an error; successful
Error
+/// objects remain ordinary values rather than entering the error channel.
#[inline]
-fn owned_from_raw(raw: TVMFFIAny) -> Result<Any> {
- if let Some(owned) = try_to_owned_without_normalization(raw) {
- return Ok(owned);
+fn raw_tvm_ffi_any_to_owned_any(raw: TVMFFIAny) -> Result<Any> {
+ if is_plain_inline(raw.type_index) {
+ return Ok(unsafe { Any::from_raw_ffi_any(raw) });
}
if raw.type_index >= TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
- return Err(runtime_error(
- "native structural map: object-backed value has a null pointer",
- ));
+ let object = unsafe { raw.data_union.v_obj };
+ if object.is_null() {
+ return Err(runtime_error(
+ "native structural map: object-backed value has a null
pointer",
+ ));
+ }
+ unsafe { object::unsafe_::inc_ref(object) };
+ return Ok(unsafe { Any::from_raw_ffi_any(raw) });
}
// Raw string/bytes views and ObjectRValueRef require normalization (or a
@@ -831,69 +907,28 @@ fn owned_from_raw(raw: TVMFFIAny) -> Result<Any> {
}
}
-#[cold]
-pub(super) fn with_mutation_error_context(error: Error, raw: TVMFFIAny) ->
Error {
- if raw.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
- error
- } else {
- with_error_context(
- with_visit_error_context(error, raw),
- &format!("object `{}`", type_key_of(raw.type_index)),
- )
- }
-}
-
-#[cold]
-fn with_error_context(error: Error, frame: &str) -> Error {
- with_structural_error_context(error, "map", frame)
-}
-
-// Error allocation is exceptional and does not belong in recursive hot frames.
-#[cold]
-#[inline(never)]
-fn runtime_error(message: &str) -> Error {
- Error::new(RUNTIME_ERROR, message, "")
-}
-
// Cache registry columns, not their optional per-type cells. Absent cells
retain
// the normal hook/reflection fallback.
-static STRUCTURAL_MUTATE_COLUMN: LazyLock<TypeAttrColumn> = LazyLock::new(|| {
+static STRUCTURAL_MUTATE_COLUMN: ColdInitLazyLock<TypeAttrColumn> =
ColdInitLazyLock::new(|| {
TypeAttrColumn::new(STRUCTURAL_MUTATE_ATTR).expect("missing __s_mutate__
column")
});
-static STRUCTURAL_MAYBE_INPLACE_MUTATE_COLUMN: LazyLock<TypeAttrColumn> =
LazyLock::new(|| {
- TypeAttrColumn::new(STRUCTURAL_MAYBE_INPLACE_MUTATE_ATTR)
- .expect("missing __s_maybe_inplace_mutate__ column")
-});
+static STRUCTURAL_MAYBE_INPLACE_MUTATE_COLUMN:
ColdInitLazyLock<TypeAttrColumn> =
+ ColdInitLazyLock::new(|| {
+ TypeAttrColumn::new(STRUCTURAL_MAYBE_INPLACE_MUTATE_ATTR)
+ .expect("missing __s_maybe_inplace_mutate__ column")
+ });
/// Return the shared column for normal or permitted in-place mutation hooks.
/// Cache the column, not its per-type cells, so absent hooks still use
fallback.
-/// The ready check stays inline; initialization uses the same LazyLock through
-/// an outlined tail to keep Once descriptors out of recursive caller frames.
#[inline(always)]
fn structural_mutate_column<const INPLACE: bool>() -> TypeAttrColumn {
- let column = if INPLACE {
- &STRUCTURAL_MAYBE_INPLACE_MUTATE_COLUMN
- } else {
- &STRUCTURAL_MUTATE_COLUMN
- };
- if let Some(column) = LazyLock::get(column) {
- *column
- } else {
- initialize_structural_mutate_column::<INPLACE>()
- }
-}
-
-// LazyLock owns blocking/poison semantics; outline its first-use Once scratch.
-#[cold]
-#[inline(never)]
-fn initialize_structural_mutate_column<const INPLACE: bool>() ->
TypeAttrColumn {
if INPLACE {
- *LazyLock::force(&STRUCTURAL_MAYBE_INPLACE_MUTATE_COLUMN)
+ *STRUCTURAL_MAYBE_INPLACE_MUTATE_COLUMN
} else {
- *LazyLock::force(&STRUCTURAL_MUTATE_COLUMN)
+ *STRUCTURAL_MUTATE_COLUMN
}
}
-static SHALLOW_COPY_COLUMN: LazyLock<TypeAttrColumn> = LazyLock::new(|| {
+static SHALLOW_COPY_COLUMN: ColdInitLazyLock<TypeAttrColumn> =
ColdInitLazyLock::new(|| {
TypeAttrColumn::new(SHALLOW_COPY_ATTR).expect("missing
__ffi_shallow_copy__ column")
});
@@ -901,6 +936,12 @@ static SHALLOW_COPY_COLUMN: LazyLock<TypeAttrColumn> =
LazyLock::new(|| {
// StructuralMutateEngine implementation
//
-----------------------------------------------------------------------------
+// Perf note: Raw carriers avoid materializing an owning Any on the success
path.
+// Private C-unwind boundaries return this 16-byte carrier in x0/x1 on AArch64,
+// avoiding the hidden result slot observed with the Rust ABI while allowing
+// Rust cleanup to unwind. Native vtables remain extern "C".
+// Raw bits provide no ownership guard: import an owning result into Any before
+// fallible work or destruction of an observer; borrowed inputs must stay live.
impl StructuralMutateEngine {
/// Recover the engine prefix for internal runtime projections.
///
@@ -971,7 +1012,7 @@ impl StructuralMutateEngine {
>,
{
let context = (*Self::from_handle(runtime)).dispatcher;
- let raw = raw_of(value);
+ let raw = *value.as_raw_ffi_any();
let mode = if INPLACE {
InplaceMode::Allow
} else {
@@ -991,7 +1032,7 @@ impl StructuralMutateEngine {
) {
// Map adapters attribute errors to their possibly replaced value;
// direct mutation attributes callback failures to this input.
- Some(result) => result_into_raw(if CALLBACK_CONTEXT {
+ Some(result) => result_into_raw_tvm_ffi_any(if CALLBACK_CONTEXT {
result.map_err(|error| with_mutation_error_context(error, raw))
} else {
result
@@ -1074,8 +1115,9 @@ impl StructuralMutateEngine {
small_str_len,
data_union,
};
- result_into_raw(
- result_from_raw(result).map_err(|error|
with_mutation_error_context(error, raw)),
+ result_into_raw_tvm_ffi_any(
+ raw_tvm_ffi_any_into_result(result)
+ .map_err(|error| with_mutation_error_context(error, raw)),
)
}
@@ -1092,7 +1134,7 @@ impl StructuralMutateEngine {
(*Self::from_handle(runtime))
.remap
.borrow()
- .get(borrowed_view(&raw))
+ .get(AnyView::from_raw_ffi_any(raw))
}
/// Record borrowed identity `raw` and clone `value` into the invocation
remap.
@@ -1109,7 +1151,7 @@ impl StructuralMutateEngine {
(*Self::from_handle(runtime))
.remap
.borrow_mut()
- .set(borrowed_view(&raw), value)
+ .set(AnyView::from_raw_ffi_any(raw), value)
}
/// Native remap lookup: borrow `var`, return an owning value, None, or
Error.
@@ -1121,10 +1163,10 @@ impl StructuralMutateEngine {
runtime: StructuralMutatorHandle,
var: AnyView<'static>,
) -> TVMFFIAny {
- match Self::runtime_var_remap_get(runtime, raw_of(var)) {
+ match Self::runtime_var_remap_get(runtime, *var.as_raw_ffi_any()) {
Ok(Some(value)) => Any::into_raw_ffi_any(value),
Ok(None) => TVMFFIAny::new(),
- Err(error) => result_into_raw(Err(error)),
+ Err(error) => result_into_raw_tvm_ffi_any(Err(error)),
}
}
@@ -1138,9 +1180,12 @@ impl StructuralMutateEngine {
var: AnyView<'static>,
replacement: AnyView<'static>,
) -> TVMFFIAny {
- result_into_raw(owned_from_raw(raw_of(replacement)).and_then(|value| {
- Self::runtime_var_remap_set(runtime, raw_of(var), &value).map(|()|
Any::new())
- }))
+ result_into_raw_tvm_ffi_any(
+
raw_tvm_ffi_any_to_owned_any(*replacement.as_raw_ffi_any()).and_then(|value| {
+ Self::runtime_var_remap_set(runtime, *var.as_raw_ffi_any(),
&value)
+ .map(|()| Any::new())
+ }),
+ )
}
/// Run the public mutate/map adapter with owned `root` and borrowed
`dispatch`.
@@ -1257,19 +1302,15 @@ impl StructuralMutateEngine {
field_changed: &mut bool,
) -> Result<()> {
let Some(getter) = field.getter else {
- return Err(runtime_error(&format!(
- "native structural map: reflected field `{}` has no getter",
- field.name.as_str()
- )));
+ return Err(missing_field_getter_error(
+ "structural map",
+ field.name.as_str(),
+ ));
};
// Read every field from the copy so earlier setters' side effects are
// visible to later field mappings, exactly as in the C++ fallback.
- let field_offset = usize::try_from(field.offset).map_err(|_| {
- runtime_error(&format!(
- "native structural map: reflected field `{}` has an invalid
offset",
- field.name.as_str()
- ))
- })?;
+ let field_offset = usize::try_from(field.offset)
+ .map_err(|_| invalid_field_offset_error(field.name.as_str()))?;
// SAFETY: registered reflection metadata guarantees that the field
// offset lies within this object's allocation. The checked conversion
// above also prevents truncation on 32-bit targets.
@@ -1278,9 +1319,10 @@ impl StructuralMutateEngine {
// populate an owning result and still report an error.
let mut child = Any::new();
if getter(source_address, Any::as_data_ptr(&mut child)) != 0 {
- return Err(with_error_context(
+ return Err(with_structural_field_error_context(
Error::from_raised(),
- &format!("field `{}`", field.name.as_str()),
+ "map",
+ field.name.as_str(),
));
}
// Reflection getters return owning values. Keep the child alive for
@@ -1290,13 +1332,13 @@ impl StructuralMutateEngine {
let mapped = Self::with_mutator_def_region(runtime, child_region, |_| {
Self::call_mutator(runtime, child_raw, InplaceMode::Disallow)
})
- .map_err(|error| with_error_context(error, &format!("field `{}`",
field.name.as_str())))?;
- if is_unchanged(&mapped) || same_shallow(child_raw,
*mapped.as_raw_ffi_any()) {
+ .map_err(|error| with_structural_field_error_context(error, "map",
field.name.as_str()))?;
+ if is_unchanged(&mapped) || raw_tvm_ffi_any_same_as(child_raw,
*mapped.as_raw_ffi_any()) {
return Ok(());
}
call_field_setter(field, source_address,
mapped.as_raw_ffi_any()).map_err(|error| {
- with_error_context(error, &format!("field `{}`",
field.name.as_str()))
+ with_structural_field_error_context(error, "map",
field.name.as_str())
})?;
*field_changed = true;
Ok(())
@@ -1321,7 +1363,7 @@ impl StructuralMutateEngine {
};
unsafe {
let view = AnyView::from_raw_ffi_any(raw);
- result_from_raw(callback(mutator, view))
+ raw_tvm_ffi_any_into_result(callback(mutator, view))
}
}
@@ -1414,14 +1456,10 @@ impl StructuralMutateEngine {
])
})
} else {
- Err(Error::new(
- TYPE_ERROR,
- "__s_mutate__ must be an opaque function pointer or
ffi.Function",
- "",
- ))
+ Err(invalid_mutation_hook_error())
};
// Preserve successful Error-valued returns until the public
conversion.
- result_into_raw(result.map_err(|error|
with_mutation_error_context(error, raw)))
+ result_into_raw_tvm_ffi_any(result.map_err(|error|
with_mutation_error_context(error, raw)))
}
/// Borrow a live runtime mutator as an object-valued ABI argument.
@@ -1507,7 +1545,8 @@ impl StructuralMutateEngine {
/// # Safety
/// `runtime` must be this live engine and the input fields a valid
borrowed value.
/// The input owner must survive reflection, remap updates, and error
annotation.
- // Outline reflection scratch without marking this normal fallback cold.
+ // Perf note: Keep reflection scratch outside recursive frames. This
fallback
+ // can be frequent, so inline(never) is intentional without cold.
// The AArch64 C ABI keeps raw results in registers; C-unwind preserves
// Rust unwinding inside this private helper.
#[inline(never)]
@@ -1517,13 +1556,15 @@ impl StructuralMutateEngine {
small_str_len: u32,
data_union: TVMFFIAnyDataUnion,
) -> TVMFFIAny {
+ // Perf note: Read region state here, before fallback semantics need
it,
+ // rather than loading it on the native-hook path that does not use it.
let def_region_kind = std::mem::transmute::<i32,
DefRegionKind>((*runtime).def_region_mode);
let raw = TVMFFIAny {
type_index,
small_str_len,
data_union,
};
- result_into_raw(
+ result_into_raw_tvm_ffi_any(
Self::mutate_without_hook(runtime, raw, def_region_kind)
.map_err(|error| with_mutation_error_context(error, raw)),
)
@@ -1539,7 +1580,7 @@ impl StructuralMutateEngine {
def_region_kind: DefRegionKind,
) -> Result<Any> {
if raw.type_index < TVMFFITypeIndex::kTVMFFIStaticObjectBegin as i32 {
- return owned_from_raw(raw);
+ return raw_tvm_ffi_any_to_owned_any(raw);
}
let type_info = checked_type_info(raw.type_index)?;
diff --git a/rust/tvm-ffi/src/extra/structural_visit.rs
b/rust/tvm-ffi/src/extra/structural_visit.rs
index 53cf9753..152dfdcc 100644
--- a/rust/tvm-ffi/src/extra/structural_visit.rs
+++ b/rust/tvm-ffi/src/extra/structural_visit.rs
@@ -46,6 +46,27 @@
//! reaching the non-unwinding `extern "C"` callback boundary aborts the
process;
//! visitor callbacks do not catch or transport panic payloads across native
frames.
+//
-----------------------------------------------------------------------------
+// Implementation overview
+//
-----------------------------------------------------------------------------
+//
+// This file is organized into:
+// - Data structures and visiting capabilities.
+// - Outlined errors and private utility functions.
+// - Engine implementation and the public entry point.
+//
+// We use narrowly scoped unsafe code for native interoperation and efficient
+// traversal. Safety and ownership contracts are documented at each boundary.
+//
+// The main performance strategies are:
+// - Outline cold work to keep its temporary storage out of recursive frames.
+// - Use _raw helpers and private extern "C-unwind" boundaries for register
+// transport on AArch64 while preserving Rust unwinding.
+// - Inline small conversions and forwarding code so ownership transfers and
+// redundant cleanup can optimize away.
+//
+// Local Perf note comments explain individual implementation choices.
+
use std::marker::PhantomData;
use super::marker::{Borrowed, Owned};
@@ -53,12 +74,12 @@ use std::ops::ControlFlow;
use std::os::raw::c_void;
use std::rc::Rc;
+use super::cold_init_lazy_lock::ColdInitLazyLock;
pub use super::structural_walk::*;
-use std::sync::LazyLock;
use crate::any::{Any, AnyView};
use crate::derive::Object;
-use crate::error::{Error, Result, RUNTIME_ERROR, TYPE_ERROR};
+use crate::error::{Error, Result, TYPE_ERROR};
use crate::function::Function;
use crate::object::{Object, ObjectArc};
use crate::reflection::TypeAttrColumn;
@@ -72,7 +93,8 @@ use crate::tvm_ffi_sys::{
};
use super::structural_common::{
- borrowed_view, raw_of, type_key_of, with_structural_error_context,
with_visit_error_context,
+ missing_field_getter_error, runtime_error, type_key_of,
with_structural_error_context,
+ with_structural_field_error_context, with_visit_error_context,
};
const STRUCTURAL_VISIT_ATTR: &str = "__s_visit__";
@@ -130,7 +152,7 @@ impl StructuralVisitorObj {
const TYPE_KEY: &'static str = "ffi.StructuralVisitor";
fn type_index() -> i32 {
- static TYPE_INDEX: LazyLock<i32> = LazyLock::new(|| unsafe {
+ static TYPE_INDEX: ColdInitLazyLock<i32> = ColdInitLazyLock::new(||
unsafe {
let key =
TVMFFIByteArray::from_str(StructuralVisitorObj::TYPE_KEY);
let mut type_index = 0;
let return_code = TVMFFITypeKeyToIndex(&key, &mut type_index);
@@ -257,13 +279,14 @@ impl<D> StructuralVisitorView<'_, D> {
where
T: Into<AnyView<'input>>,
{
+ let value: AnyView<'input> = value.into();
// SAFETY: the view ties this runtime to D; the check establishes the
// instance, and publishing uses the caller's actual exclusive
reborrow.
unsafe {
visit_result_from_any(Any::from_raw_ffi_any(StructuralVisitEngine::visit_raw(
self.runtime,
dispatch,
- raw_of(value.into()),
+ *value.as_raw_ffi_any(),
)))
}
}
@@ -283,11 +306,12 @@ impl<D> StructuralVisitorView<'_, D> {
// the supplied borrow must identify this runtime's dispatcher
instance.
unsafe {
let context =
StructuralVisitEngine::checked_dispatcher(self.runtime, dispatch);
+ let value: AnyView<'input> = value.into();
visit_result_from_any(Any::from_raw_ffi_any(
StructuralVisitEngine::default_visit_raw(
self.runtime,
context,
- raw_of(value.into()),
+ *value.as_raw_ffi_any(),
),
))
}
@@ -311,7 +335,7 @@ where
fn visit_root(mut self, root: AnyView<'_>) ->
Result<Option<VisitInterrupt>> {
visit_result_from_any(unsafe {
Any::from_raw_ffi_any(StructuralVisitEngine::run_visitor_raw::<D>(
- raw_of(root),
+ *root.as_raw_ffi_any(),
&mut self,
))
})
@@ -329,7 +353,7 @@ where
fn visit_root(self, root: AnyView<'_>) -> Result<Option<VisitInterrupt>> {
visit_result_from_any(unsafe {
Any::from_raw_ffi_any(StructuralVisitEngine::run_visitor_raw::<D>(
- raw_of(root),
+ *root.as_raw_ffi_any(),
self,
))
})
@@ -337,21 +361,68 @@ where
}
//
-----------------------------------------------------------------------------
-// Private utility functions
+// Outlined errors
//
-----------------------------------------------------------------------------
+// Keep construction and formatting scratch outside successful traversal
frames.
-#[inline(always)]
-fn dispatcher_context<D>(dispatch: &mut D) -> *mut c_void {
- std::ptr::from_mut(dispatch).cast()
+#[cold]
+#[inline(never)]
+fn unregistered_visit_type_error(type_index: i32) -> Error {
+ runtime_error(&format!(
+ "native visitor: unregistered type index {type_index}"
+ ))
+}
+
+#[cold]
+#[inline(never)]
+fn invalid_visit_hook_result_error() -> Error {
+ Error::new(
+ TYPE_ERROR,
+ "structural visit hook must return None or ffi.VisitInterrupt",
+ "",
+ )
}
-// Keep panic construction out of the successful dispatcher identity check.
+/// Consume `error` and annotate it from borrowed input scalar fields. Internal
+/// callers keep that input live; the Error remains owned if formatting
unwinds.
+// Perf note: Keep error formatting and context-allocation scratch out of
visit frames.
+#[cold]
+#[inline(never)]
+fn annotate_visit_error(
+ error: Error,
+ type_index: i32,
+ small_str_len: u32,
+ data_union: TVMFFIAnyDataUnion,
+) -> Error {
+ let value = TVMFFIAny {
+ type_index,
+ small_str_len,
+ data_union,
+ };
+ let error = with_visit_error_context(error, value);
+ with_structural_error_context(
+ error,
+ "visit",
+ &format!("object `{}`", type_key_of(value.type_index)),
+ )
+}
+
+// Perf note: Outline panic construction; the successful identity check stays
small.
#[cold]
#[inline(never)]
fn wrong_visitor_dispatcher() -> ! {
panic!("structural visitor called with a different dispatcher instance")
}
+//
-----------------------------------------------------------------------------
+// Private utility functions
+//
-----------------------------------------------------------------------------
+
+#[inline(always)]
+fn dispatcher_context<D>(dispatch: &mut D) -> *mut c_void {
+ std::ptr::from_mut(dispatch).cast()
+}
+
#[inline(always)]
fn visit_result_raw(result: Result<Option<VisitInterrupt>>, raw: TVMFFIAny) ->
TVMFFIAny {
match result {
@@ -377,12 +448,7 @@ fn checked_hook_result(value: Any) -> Any {
{
value
} else {
- Error::new(
- TYPE_ERROR,
- "structural visit hook must return None or ffi.VisitInterrupt",
- "",
- )
- .into()
+ invalid_visit_hook_result_error().into()
}
}
@@ -427,46 +493,26 @@ pub(crate) fn free_var_child_region(
}
#[inline(always)]
-fn with_error_context(result: Any, frame: &str) -> Any {
+fn with_field_error_context(result: Any, name: &str) -> Any {
if result.type_index() == TVMFFITypeIndex::kTVMFFIError as i32 {
core::hint::cold_path();
// SAFETY: direct tag check establishes the Error representation.
- with_structural_error_context(unsafe { take_error(result) }, "visit",
frame).into()
+ with_structural_field_error_context(unsafe { take_error(result) },
"visit", name).into()
} else {
result
}
}
-// Error allocation is exceptional and does not belong in recursive hot frames.
-#[cold]
-#[inline(never)]
-fn runtime_error(message: &str) -> Error {
- Error::new(RUNTIME_ERROR, message, "")
-}
-
// Columns are process-lifetime registry entries; a missing per-type cell still
// falls back to reflected traversal. A missing required column is an error.
-static STRUCTURAL_VISIT_COLUMN: LazyLock<TypeAttrColumn> = LazyLock::new(|| {
+static STRUCTURAL_VISIT_COLUMN: ColdInitLazyLock<TypeAttrColumn> =
ColdInitLazyLock::new(|| {
TypeAttrColumn::new(STRUCTURAL_VISIT_ATTR).expect("missing __s_visit__
column")
});
/// Return the shared visit-hook column without caching optional per-type
cells.
-/// The ready check stays inline; first use forces the same LazyLock in an
-/// outlined tail, preserving blocking/poison behavior without caller Once
scratch.
#[inline(always)]
fn structural_visit_column() -> TypeAttrColumn {
- if let Some(column) = LazyLock::get(&STRUCTURAL_VISIT_COLUMN) {
- *column
- } else {
- initialize_structural_visit_column()
- }
-}
-
-// Keep Once's initialization descriptors out of each recursive caller's frame.
-#[cold]
-#[inline(never)]
-fn initialize_structural_visit_column() -> TypeAttrColumn {
- *LazyLock::force(&STRUCTURAL_VISIT_COLUMN)
+ *STRUCTURAL_VISIT_COLUMN
}
/// Visit every reflected field described by `info` and its ancestors in the
@@ -512,9 +558,13 @@ unsafe fn visit_field_level<B>(
// StructuralVisitEngine implementation
//
-----------------------------------------------------------------------------
-// Operate in the raw path to optimize stack frame cost. Each `_raw` result
-// carries one owned ABI value: forward it exactly once or import it into Any
-// before any operation that can unwind or discard it.
+// Perf note: Raw carriers avoid materializing an owning Any on the success
path.
+// Private C-unwind boundaries return this 16-byte carrier in x0/x1 on AArch64,
+// avoiding the hidden result slot observed with the Rust ABI while allowing
+// Rust cleanup to unwind. Native vtables remain extern "C".
+// Raw bits provide no ownership guard: each `_raw` result carries one owned
ABI
+// value. Forward it exactly once or import it into Any before any operation
+// that can unwind or discard it, including destruction of an observer.
impl StructuralVisitEngine {
/// Consume owning `result` from descent, adding borrowed `value` context
on Error.
/// Internal callers supply valid tokens and keep the input live during
annotation;
@@ -531,7 +581,7 @@ impl StructuralVisitEngine {
unsafe {
let error = take_error(Any::from_raw_ffi_any(result));
Any::into_raw_ffi_any(
- Self::annotate_visit_error(
+ annotate_visit_error(
error,
value.type_index,
value.small_str_len,
@@ -543,30 +593,6 @@ impl StructuralVisitEngine {
}
}
- /// Consume `error` and annotate it from borrowed input scalar fields.
Internal
- /// callers keep that input live; the Error remains owned if formatting
unwinds.
- // Keep error formatting and context-allocation scratch out of visit
frames.
- #[cold]
- #[inline(never)]
- fn annotate_visit_error(
- error: Error,
- type_index: i32,
- small_str_len: u32,
- data_union: TVMFFIAnyDataUnion,
- ) -> Error {
- let value = TVMFFIAny {
- type_index,
- small_str_len,
- data_union,
- };
- let error = with_visit_error_context(error, value);
- with_structural_error_context(
- error,
- "visit",
- &format!("object `{}`", type_key_of(value.type_index)),
- )
- }
-
/// Recover the Rust engine whose first field is this ABI base.
///
/// # Safety
@@ -615,7 +641,8 @@ impl StructuralVisitEngine {
/// # Safety
/// The input fields and runtime must be valid for synchronous descent;
registered
/// reflection metadata must satisfy its getter/offset contracts.
- // Outline reflection scratch without marking this normal fallback cold.
+ // Perf note: Keep reflection scratch outside recursive frames. This
fallback
+ // can be frequent, so inline(never) is intentional without cold.
// The AArch64 C ABI returns TVMFFIAny in registers; C-unwind preserves
// Rust cleanup if reflected traversal panics inside this private helper.
#[inline(never)]
@@ -632,13 +659,7 @@ impl StructuralVisitEngine {
};
let type_info = unsafe { TVMFFIGetTypeInfo(value.type_index) };
if type_info.is_null() {
- return Any::into_raw_ffi_any(
- runtime_error(&format!(
- "native visitor: unregistered type index {}",
- value.type_index
- ))
- .into(),
- );
+ return
Any::into_raw_ffi_any(unregistered_visit_type_error(value.type_index).into());
}
let seq_hash_kind = unsafe {
let metadata = (*type_info).metadata;
@@ -681,28 +702,22 @@ impl StructuralVisitEngine {
}
let Some(getter) = field.getter else {
- return Any::from(runtime_error(&format!(
- "native visitor: reflected field `{}` has no getter",
- field.name.as_str()
- )));
+ return Any::from(missing_field_getter_error("visitor",
field.name.as_str()));
};
let address = object.offset(field.offset as isize) as *mut c_void;
// Own the getter result so partial writes and recursive borrows drop
safely.
let mut child = Any::new();
if getter(address, Any::as_data_ptr(&mut child)) != 0 {
- return with_error_context(
- Any::from(Error::from_raised()),
- &format!("field `{}`", field.name.as_str()),
- );
+ return with_field_error_context(Any::from(Error::from_raised()),
field.name.as_str());
}
let borrowed = AnyView::from(&child);
let child_region = field_def_region(field, inherited_region);
- with_error_context(
+ with_field_error_context(
Self::with_visitor_def_region(runtime, child_region, || {
- Any::from_raw_ffi_any(Self::call_visitor_raw(runtime,
raw_of(borrowed)))
+ Any::from_raw_ffi_any(Self::call_visitor_raw(runtime,
*borrowed.as_raw_ffi_any()))
}),
- &format!("field `{}`", field.name.as_str()),
+ field.name.as_str(),
)
}
@@ -783,9 +798,11 @@ impl StructuralVisitEngine {
} else {
let dispatch = &mut *context.cast::<D>();
let mut visitor = Self::visitor_context::<D>(runtime);
+ // SAFETY: the native caller keeps the input alive for this
synchronous
+ // dispatch; the higher-ranked callback cannot retain its borrowed
view.
match crate::Dispatcher::dispatch_tuple(
&mut *dispatch,
- (borrowed_view(&raw), &mut visitor),
+ (AnyView::from_raw_ffi_any(raw), &mut visitor),
) {
Some(result) => visit_result_raw(result, raw),
None => Self::default_visit_raw(runtime,
dispatcher_context(dispatch), raw),
diff --git a/rust/tvm-ffi/src/extra/structural_walk.rs
b/rust/tvm-ffi/src/extra/structural_walk.rs
index d95ba5a1..08d0283c 100644
--- a/rust/tvm-ffi/src/extra/structural_walk.rs
+++ b/rust/tvm-ffi/src/extra/structural_walk.rs
@@ -22,7 +22,7 @@
//! Callbacks report failures with [`Result`] and must not panic. A panic
reaching
//! the non-unwinding native callback boundary aborts the process.
-use super::structural_common::{impl_callback_chain_tuple_arities, raw_of};
+use super::structural_common::impl_callback_chain_tuple_arities;
use super::structural_visit::{
visit_result_from_any, DefRegionKind, StructuralVisitEngine,
StructuralVisitorView,
VisitInterrupt,
@@ -540,7 +540,7 @@ fn run_walker<D>(root: AnyView<'_>, observer: D, order:
WalkOrder) -> Result<Opt
where
for<'s, 'v> &'s mut D: Dispatcher<(AnyView<'v>, DefRegionKind),
Option<Result<WalkResult>>>,
{
- let root = raw_of(root);
+ let root = *root.as_raw_ffi_any();
// Own the result while dropping the observer: user capture destructors can
// unwind after the native callback has returned.
let result = match order {
diff --git a/rust/tvm-ffi/src/extra/unchanged.rs
b/rust/tvm-ffi/src/extra/unchanged.rs
index 07f3223a..631f7237 100644
--- a/rust/tvm-ffi/src/extra/unchanged.rs
+++ b/rust/tvm-ffi/src/extra/unchanged.rs
@@ -21,7 +21,7 @@
use std::marker::PhantomData;
-use super::structural_common::structural_error_from_raw;
+use super::structural_common::{raw_tvm_ffi_any_same_as,
structural_error_from_raw};
use crate::any::{Any, AnyView, TryFromTemp};
use crate::error::{Error, Result, TYPE_ERROR};
@@ -90,7 +90,7 @@ impl<T: ContainerElement> UnchangedOr<T> {
unsafe {
T::container_copy_to_any_view(original, &mut raw);
}
- super::structural_common::same_shallow(raw,
*self.data.as_raw_ffi_any())
+ raw_tvm_ffi_any_same_as(raw, *self.data.as_raw_ffi_any())
}
/// Move out the replacement, returning None when unchanged.
diff --git a/rust/tvm-ffi/src/macros.rs b/rust/tvm-ffi/src/macros.rs
index 8c88fba6..d53f4b0c 100644
--- a/rust/tvm-ffi/src/macros.rs
+++ b/rust/tvm-ffi/src/macros.rs
@@ -49,12 +49,13 @@ macro_rules! function_name {
#[macro_export]
macro_rules! cached_global_func {
($name:literal) => {{
- static FUNC: std::sync::LazyLock<$crate::function::Function> =
- std::sync::LazyLock::new(|| {
-
$crate::function::Function::get_global($name).unwrap_or_else(|_| {
- panic!(concat!("global function `", $name, "` is not
registered"))
- })
- });
+ static FUNC: $crate::extra::cold_init_lazy_lock::ColdInitLazyLock<
+ $crate::function::Function,
+ > = $crate::extra::cold_init_lazy_lock::ColdInitLazyLock::new(|| {
+ $crate::function::Function::get_global($name).unwrap_or_else(|_| {
+ panic!(concat!("global function `", $name, "` is not
registered"))
+ })
+ });
&*FUNC
}};
}
