This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm-ffi.git
The following commit(s) were added to refs/heads/main by this push:
new e74e58f9 [FFI][PERF] Force-inline ObjectPtr wrapper special members
(#761)
e74e58f9 is described below
commit e74e58f9d8a6fb1ece5722d0715f68187a8e0385
Author: Tianqi Chen <[email protected]>
AuthorDate: Mon Sep 7 17:24:05 2026 -0400
[FFI][PERF] Force-inline ObjectPtr wrapper special members (#761)
Force-inline all five special members of the single-payload wrappers
that transitively own `ObjectPtr` storage.
This keeps moved-from destructor cleanup within hot callers even after
the compiler exhausts its ordinary inlining budget, while explicitly
retaining noexcept move construction and assignment.
---
include/tvm/ffi/container/variant.h | 10 ++++++----
include/tvm/ffi/expected.h | 27 +++++++++++++++++++++++++++
include/tvm/ffi/object.h | 32 ++++++++++++++++++--------------
include/tvm/ffi/optional.h | 32 ++++++++++++++++++++------------
4 files changed, 71 insertions(+), 30 deletions(-)
diff --git a/include/tvm/ffi/container/variant.h
b/include/tvm/ffi/container/variant.h
index 1823da42..494156d7 100644
--- a/include/tvm/ffi/container/variant.h
+++ b/include/tvm/ffi/container/variant.h
@@ -60,28 +60,30 @@ class Variant {
template <typename T>
using enable_if_variant_contains_t = std::enable_if_t<variant_contains_v<T>>;
/// \endcond
+ // Special members are explicitly inlined to enable move cleanup
optimizations
+ TVM_FFI_INLINE ~Variant() = default;
/*!
* \brief Constructor from another variant
* \param other The other variant
*/
- Variant(const Variant<V...>& other) = default;
+ TVM_FFI_INLINE Variant(const Variant<V...>& other) = default;
/*!
* \brief Constructor from another variant
* \param other The other variant
*/
- Variant(Variant<V...>&& other) noexcept = default;
+ TVM_FFI_INLINE Variant(Variant<V...>&& other) noexcept = default;
/*!
* \brief Assignment from another variant
* \param other The other variant
*/
- Variant& operator=(const Variant<V...>& other) = default;
+ TVM_FFI_INLINE Variant& operator=(const Variant<V...>& other) = default;
/*!
* \brief Assignment from another variant
* \param other The other variant
*/
- Variant& operator=(Variant<V...>&& other) noexcept = default;
+ TVM_FFI_INLINE Variant& operator=(Variant<V...>&& other) noexcept = default;
/*!
* \brief Constructor from a contained value
diff --git a/include/tvm/ffi/expected.h b/include/tvm/ffi/expected.h
index 6a7e3f2f..ffe91a25 100644
--- a/include/tvm/ffi/expected.h
+++ b/include/tvm/ffi/expected.h
@@ -43,6 +43,15 @@ class Unexpected {
"Unexpected<E> requires E to be Error or a subclass of
Error.");
public:
+ // Special members are explicitly inlined to enable move cleanup
optimizations
+ TVM_FFI_INLINE ~Unexpected() = default;
+ /// \cond Doxygen_Suppress
+ TVM_FFI_INLINE Unexpected(const Unexpected&) = default;
+ TVM_FFI_INLINE Unexpected(Unexpected&&) noexcept = default;
+ /// \endcond
+ TVM_FFI_INLINE Unexpected& operator=(const Unexpected&) = default;
+ TVM_FFI_INLINE Unexpected& operator=(Unexpected&&) noexcept = default;
+
/*! \brief Construct from an error value. */
explicit Unexpected(E error) : error_(std::move(error)) {}
@@ -119,6 +128,15 @@ class Expected {
"Expected with a cv-qualified void success type is not allowed. Use
Expected<void>.");
static_assert(!std::is_same_v<T, Error>, "Expected<Error> is not allowed.
Use Error directly.");
+ // Special members are explicitly inlined to enable move cleanup
optimizations
+ TVM_FFI_INLINE ~Expected() = default;
+ /// \cond Doxygen_Suppress
+ TVM_FFI_INLINE Expected(const Expected&) = default;
+ TVM_FFI_INLINE Expected(Expected&&) noexcept = default;
+ /// \endcond
+ TVM_FFI_INLINE Expected& operator=(const Expected&) = default;
+ TVM_FFI_INLINE Expected& operator=(Expected&&) noexcept = default;
+
/*!
* \brief Implicit constructor from a success value.
* \param value The success value.
@@ -279,6 +297,15 @@ class Expected {
template <>
class Expected<void> {
public:
+ // Special members are explicitly inlined to enable move cleanup
optimizations
+ TVM_FFI_INLINE ~Expected() = default;
+ /// \cond Doxygen_Suppress
+ TVM_FFI_INLINE Expected(const Expected&) = default;
+ TVM_FFI_INLINE Expected(Expected&&) noexcept = default;
+ /// \endcond
+ TVM_FFI_INLINE Expected& operator=(const Expected&) = default;
+ TVM_FFI_INLINE Expected& operator=(Expected&&) noexcept = default;
+
/*! \brief Construct a successful Expected<void>. */
Expected() = default;
diff --git a/include/tvm/ffi/object.h b/include/tvm/ffi/object.h
index 5ca3c043..5c7832d3 100644
--- a/include/tvm/ffi/object.h
+++ b/include/tvm/ffi/object.h
@@ -416,8 +416,7 @@ class Object {
template <typename T>
class ObjectPtr {
public:
- // Core value members that perform ownership work are force-inlined so
refcount paths do not
- // depend on compiler cost models. Defaulted and empty members intentionally
stay unmarked.
+ // Special members are explicitly inlined to enable move cleanup
optimizations
/*! \brief default constructor */
ObjectPtr() = default;
/*! \brief default constructor */
@@ -426,14 +425,14 @@ class ObjectPtr {
* \brief copy constructor
* \param other The value to be moved
*/
- ObjectPtr(const ObjectPtr<T>& other) // NOLINT(*)
+ TVM_FFI_INLINE ObjectPtr(const ObjectPtr<T>& other) // NOLINT(*)
: ObjectPtr(other.data_) {}
/*!
* \brief copy constructor
* \param other The value to be moved
*/
template <typename U>
- ObjectPtr(const ObjectPtr<U>& other) // NOLINT(*)
+ TVM_FFI_INLINE ObjectPtr(const ObjectPtr<U>& other) // NOLINT(*)
: ObjectPtr(other.data_) {
static_assert(std::is_base_of_v<T, U>, "can only assign of child class
ObjectPtr to parent");
}
@@ -441,7 +440,7 @@ class ObjectPtr {
* \brief move constructor
* \param other The value to be moved
*/
- TVM_FFI_INLINE ObjectPtr(ObjectPtr<T>&& other) // NOLINT(*)
+ TVM_FFI_INLINE ObjectPtr(ObjectPtr<T>&& other) noexcept // NOLINT(*)
: data_(other.data_) {
other.data_ = nullptr;
}
@@ -450,7 +449,7 @@ class ObjectPtr {
* \param other The value to be moved
*/
template <typename Y>
- TVM_FFI_INLINE ObjectPtr(ObjectPtr<Y>&& other) // NOLINT(*)
+ TVM_FFI_INLINE ObjectPtr(ObjectPtr<Y>&& other) noexcept // NOLINT(*)
: data_(other.data_) {
static_assert(std::is_base_of_v<T, Y>, "can only assign of child class
ObjectPtr to parent");
other.data_ = nullptr;
@@ -483,7 +482,7 @@ class ObjectPtr {
* \param other The value to be assigned.
* \return reference to self.
*/
- ObjectPtr<T>& operator=(const ObjectPtr<T>& other) { // NOLINT(*)
+ TVM_FFI_INLINE ObjectPtr<T>& operator=(const ObjectPtr<T>& other) { //
NOLINT(*)
// takes in plane operator to enable copy elison.
// copy-and-swap idiom
ObjectPtr(other).swap(*this); // NOLINT(*)
@@ -494,7 +493,7 @@ class ObjectPtr {
* \param other The value to be assigned.
* \return reference to self.
*/
- TVM_FFI_INLINE ObjectPtr<T>& operator=(ObjectPtr<T>&& other) { // NOLINT(*)
+ TVM_FFI_INLINE ObjectPtr<T>& operator=(ObjectPtr<T>&& other) noexcept { //
NOLINT(*)
// copy-and-swap idiom
ObjectPtr(std::move(other)).swap(*this); // NOLINT(*)
return *this;
@@ -564,17 +563,20 @@ class Arc : public ObjectPtr<T> {
Arc() = delete;
Arc(std::nullptr_t) = delete;
+ // Special members are explicitly inlined to enable move cleanup
optimizations
+ TVM_FFI_INLINE ~Arc() = default;
+
/*! \brief Copy constructor. */
- Arc(const Arc&) = default;
+ TVM_FFI_INLINE Arc(const Arc&) = default;
/*! \brief Move constructor. */
- Arc(Arc&&) = default;
+ TVM_FFI_INLINE Arc(Arc&&) noexcept = default;
/*! \brief Copy assignment operator. */
- Arc& operator=(const Arc&) = default;
+ TVM_FFI_INLINE Arc& operator=(const Arc&) = default;
/*! \brief Move assignment operator. */
- Arc& operator=(Arc&&) = default;
+ TVM_FFI_INLINE Arc& operator=(Arc&&) noexcept = default;
/*!
* \brief Copy-upcast an Arc of a derived Object type.
@@ -810,14 +812,16 @@ class ObjectRef {
public:
/*! \brief default constructor */
ObjectRef() = default;
+ // Special members are explicitly inlined to enable move cleanup
optimizations
+ TVM_FFI_INLINE ~ObjectRef() = default;
/*! \brief copy constructor */
- ObjectRef(const ObjectRef& other) = default;
+ TVM_FFI_INLINE ObjectRef(const ObjectRef& other) = default;
/*! \brief move constructor */
TVM_FFI_INLINE ObjectRef(ObjectRef&& other) noexcept :
data_(std::move(other.data_)) {
other.data_ = nullptr;
}
/*! \brief copy assignment */
- ObjectRef& operator=(const ObjectRef& other) = default;
+ TVM_FFI_INLINE ObjectRef& operator=(const ObjectRef& other) = default;
/*! \brief move assignment */
TVM_FFI_INLINE ObjectRef& operator=(ObjectRef&& other) noexcept {
data_ = std::move(other.data_);
diff --git a/include/tvm/ffi/optional.h b/include/tvm/ffi/optional.h
index 2b8dbd15..24ea229b 100644
--- a/include/tvm/ffi/optional.h
+++ b/include/tvm/ffi/optional.h
@@ -83,15 +83,17 @@ class Optional<T,
public:
// default constructors.
Optional() = default;
+ // Special members are explicitly inlined to enable move cleanup
optimizations
+ TVM_FFI_INLINE ~Optional() = default;
// NOLINTBEGIN(google-explicit-constructor)
- Optional(const Optional& other) = default;
- Optional(Optional&& other) noexcept = default;
+ TVM_FFI_INLINE Optional(const Optional& other) = default;
+ TVM_FFI_INLINE Optional(Optional&& other) noexcept = default;
TVM_FFI_INLINE Optional(std::optional<T> other) : data_(std::move(other)) {}
Optional(std::nullopt_t) {}
TVM_FFI_INLINE Optional(T other) : data_(std::move(other)) {}
// NOLINTEND(google-explicit-constructor)
- Optional& operator=(const Optional& other) = default;
+ TVM_FFI_INLINE Optional& operator=(const Optional& other) = default;
TVM_FFI_INLINE Optional& operator=(Optional&& other) noexcept {
data_ = std::move(other.data_);
return *this;
@@ -180,13 +182,15 @@ class Optional<T,
public:
/*! \brief default constructor, represents nullopt (Any() is kTVMFFINone). */
Optional() = default;
+ // Special members are explicitly inlined to enable move cleanup
optimizations
+ TVM_FFI_INLINE ~Optional() = default;
// NOLINTBEGIN(google-explicit-constructor)
/*! \brief construct nullopt from std::nullopt. */
Optional(std::nullopt_t) {}
/*! \brief copy constructor. */
- Optional(const Optional& other) = default;
+ TVM_FFI_INLINE Optional(const Optional& other) = default;
/*! \brief move constructor. */
- Optional(Optional&& other) noexcept = default;
+ TVM_FFI_INLINE Optional(Optional&& other) noexcept = default;
/*! \brief construct from a value of type T (copy). */
TVM_FFI_INLINE Optional(const T& value) : data_(value) {}
/*! \brief construct from a value of type T (move). */
@@ -200,7 +204,7 @@ class Optional<T,
// NOLINTEND(google-explicit-constructor)
/*! \brief copy assignment. */
- Optional& operator=(const Optional& other) = default;
+ TVM_FFI_INLINE Optional& operator=(const Optional& other) = default;
/*! \brief move assignment. */
TVM_FFI_INLINE Optional& operator=(Optional&& other) noexcept {
data_ = std::move(other.data_);
@@ -375,9 +379,11 @@ class Optional<T,
std::enable_if_t<use_object_ref_optional_v<T>>> : public Objec
static constexpr bool _type_container_is_exact = T::_type_container_is_exact;
Optional() = default;
+ // Special members are explicitly inlined to enable move cleanup
optimizations
+ TVM_FFI_INLINE ~Optional() = default;
// NOLINTBEGIN(google-explicit-constructor)
- Optional(const Optional&) = default;
- Optional(Optional&&) noexcept = default;
+ TVM_FFI_INLINE Optional(const Optional&) = default;
+ TVM_FFI_INLINE Optional(Optional&&) noexcept = default;
explicit Optional(UnsafeInit tag) : ObjectRef(tag) {}
Optional(std::nullopt_t) {}
Optional(std::nullptr_t) {}
@@ -389,7 +395,7 @@ class Optional<T,
std::enable_if_t<use_object_ref_optional_v<T>>> : public Objec
TVM_FFI_INLINE Optional(T other) : ObjectRef(std::move(other)) {}
// NOLINTEND(google-explicit-constructor)
- Optional& operator=(const Optional&) = default;
+ TVM_FFI_INLINE Optional& operator=(const Optional&) = default;
TVM_FFI_INLINE Optional& operator=(Optional&& other) noexcept {
ObjectRef::operator=(std::move(other));
return *this;
@@ -552,9 +558,11 @@ class Optional<T, std::enable_if_t<is_object_ptr_type_v<T>
|| is_arc_type_v<T>>>
using ContainerType = typename Traits::ContainerType;
Optional() = default;
+ // Special members are explicitly inlined to enable move cleanup
optimizations
+ TVM_FFI_INLINE ~Optional() = default;
// NOLINTBEGIN(google-explicit-constructor)
- Optional(const Optional&) = default;
- Optional(Optional&&) noexcept = default;
+ TVM_FFI_INLINE Optional(const Optional&) = default;
+ TVM_FFI_INLINE Optional(Optional&&) noexcept = default;
Optional(std::nullopt_t) : StorageType(nullptr) {}
Optional(std::nullptr_t) : StorageType(nullptr) {}
TVM_FFI_INLINE Optional(std::optional<T> other) {
@@ -565,7 +573,7 @@ class Optional<T, std::enable_if_t<is_object_ptr_type_v<T>
|| is_arc_type_v<T>>>
TVM_FFI_INLINE Optional(T value) : StorageType(std::move(value)) {}
// NOLINTEND(google-explicit-constructor)
- Optional& operator=(const Optional&) = default;
+ TVM_FFI_INLINE Optional& operator=(const Optional&) = default;
TVM_FFI_INLINE Optional& operator=(Optional&& other) noexcept {
static_cast<StorageType&>(*this) =
std::move(static_cast<StorageType&>(other));
return *this;
