This is an automated email from the ASF dual-hosted git repository.
tlopex pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm-ffi.git
The following commit(s) were added to refs/heads/main by this push:
new 0909bb4b [FFI][PERF] Mark non-trivial special members TVM_FFI_INLINE
(#759)
0909bb4b is described below
commit 0909bb4bdefa07b885b3750ea08a3b099c3c4674
Author: Tianqi Chen <[email protected]>
AuthorDate: Sun Sep 6 22:45:36 2026 -0400
[FFI][PERF] Mark non-trivial special members TVM_FFI_INLINE (#759)
Sometimes clang forgets to inline some constructors, which prevents
useful optimization.
---
include/tvm/ffi/any.h | 12 ++++++------
include/tvm/ffi/expected.h | 14 +++++++-------
include/tvm/ffi/object.h | 20 ++++++++++++--------
include/tvm/ffi/optional.h | 40 ++++++++++++++++++++++++++--------------
4 files changed, 51 insertions(+), 35 deletions(-)
diff --git a/include/tvm/ffi/any.h b/include/tvm/ffi/any.h
index 219ab47a..921feda6 100644
--- a/include/tvm/ffi/any.h
+++ b/include/tvm/ffi/any.h
@@ -73,7 +73,7 @@ class AnyView {
/*! \return the internal type index */
TVM_FFI_INLINE int32_t type_index() const noexcept { return
data_.type_index; }
/*! \brief Default constructor */
- AnyView() {
+ TVM_FFI_INLINE AnyView() {
data_.type_index = TypeIndex::kTVMFFINone;
data_.zero_padding = 0;
data_.v_int64 = 0;
@@ -93,7 +93,7 @@ class AnyView {
* \param other The value to convert from.
*/
template <typename T, typename =
std::enable_if_t<TypeTraits<T>::convert_enabled>>
- AnyView(const T& other) { // NOLINT(*)
+ TVM_FFI_INLINE AnyView(const T& other) { // NOLINT(*)
TypeTraits<T>::CopyToAnyView(other, &data_);
}
/*!
@@ -258,7 +258,7 @@ class Any {
/*!
* \brief Default constructor
*/
- Any() {
+ TVM_FFI_INLINE Any() {
data_.type_index = TypeIndex::kTVMFFINone;
data_.zero_padding = 0;
data_.v_int64 = 0;
@@ -266,12 +266,12 @@ class Any {
/*!
* \brief Destructor
*/
- ~Any() { this->reset(); }
+ TVM_FFI_INLINE ~Any() { this->reset(); }
/*!
* \brief Constructor from another Any
* \param other The other Any
*/
- Any(const Any& other) : data_(other.data_) {
+ TVM_FFI_INLINE Any(const Any& other) : data_(other.data_) {
if (data_.type_index >= TypeIndex::kTVMFFIStaticObjectBegin) {
details::ObjectUnsafe::IncRefObjectHandle(data_.v_obj);
}
@@ -280,7 +280,7 @@ class Any {
* \brief Move constructor from another Any
* \param other The other Any
*/
- Any(Any&& other) noexcept : data_(other.data_) {
+ TVM_FFI_INLINE Any(Any&& other) noexcept : data_(other.data_) {
other.data_.type_index = TypeIndex::kTVMFFINone;
other.data_.zero_padding = 0;
other.data_.v_int64 = 0;
diff --git a/include/tvm/ffi/expected.h b/include/tvm/ffi/expected.h
index 6bf6051f..6a7e3f2f 100644
--- a/include/tvm/ffi/expected.h
+++ b/include/tvm/ffi/expected.h
@@ -124,7 +124,7 @@ class Expected {
* \param value The success value.
*/
// NOLINTNEXTLINE(google-explicit-constructor,runtime/explicit)
- Expected(T value) : data_(Any(std::move(value))) {}
+ TVM_FFI_INLINE Expected(T value) : data_(Any(std::move(value))) {}
/*!
* \brief Implicit constructor from a different success value type.
@@ -144,7 +144,7 @@ class Expected {
!std::is_base_of_v<Error,
std::decay_t<U>> &&
std::is_convertible_v<U,
T>>>
// NOLINTNEXTLINE(google-explicit-constructor,runtime/explicit)
- Expected(U&& value) : data_(Any(T(std::forward<U>(value)))) {}
+ TVM_FFI_INLINE Expected(U&& value) : data_(Any(T(std::forward<U>(value)))) {}
/*!
* \brief Implicit converting constructor from another Expected success type.
@@ -160,7 +160,7 @@ class Expected {
typename = std::enable_if_t<!std::is_void_v<U> &&
(type_subsumes_v<T, U> ||
std::is_convertible_v<U, T>)>>
// NOLINTNEXTLINE(google-explicit-constructor,runtime/explicit)
- Expected(Expected<U> other) {
+ TVM_FFI_INLINE Expected(Expected<U> other) {
if constexpr (type_subsumes_v<T, U>) {
// data_ holds a T or an Error. Subsumption proves the source
representation already
// satisfies that invariant, so the Any moves without inspecting its
state. Do not make
@@ -177,12 +177,12 @@ class Expected {
* \param error The error value.
*/
// NOLINTNEXTLINE(google-explicit-constructor,runtime/explicit)
- Expected(Error error) : data_(Any(std::move(error))) {}
+ TVM_FFI_INLINE Expected(Error error) : data_(Any(std::move(error))) {}
/*! \brief Implicit constructor from an Unexpected wrapper. */
template <typename E, typename = std::enable_if_t<std::is_base_of_v<Error,
std::remove_cv_t<E>>>>
// NOLINTNEXTLINE(google-explicit-constructor,runtime/explicit)
- Expected(Unexpected<E> unexpected) :
data_(Any(std::move(unexpected).error())) {}
+ TVM_FFI_INLINE Expected(Unexpected<E> unexpected) :
data_(Any(std::move(unexpected).error())) {}
/*! \brief Return the raw stored type index. */
TVM_FFI_INLINE int32_t type_index() const noexcept { return
data_.type_index(); }
@@ -287,12 +287,12 @@ class Expected<void> {
* \param error The error value.
*/
// NOLINTNEXTLINE(google-explicit-constructor,runtime/explicit)
- Expected(Error error) : data_(Any(std::move(error))) {}
+ TVM_FFI_INLINE Expected(Error error) : data_(Any(std::move(error))) {}
/*! \brief Implicit constructor from an Unexpected wrapper. */
template <typename E, typename = std::enable_if_t<std::is_base_of_v<Error,
std::remove_cv_t<E>>>>
// NOLINTNEXTLINE(google-explicit-constructor,runtime/explicit)
- Expected(Unexpected<E> unexpected) :
data_(Any(std::move(unexpected).error())) {}
+ TVM_FFI_INLINE Expected(Unexpected<E> unexpected) :
data_(Any(std::move(unexpected).error())) {}
/*! \brief Return the raw stored type index. */
TVM_FFI_INLINE int32_t type_index() const noexcept { return
data_.type_index(); }
diff --git a/include/tvm/ffi/object.h b/include/tvm/ffi/object.h
index d8a80efc..8c442ab1 100644
--- a/include/tvm/ffi/object.h
+++ b/include/tvm/ffi/object.h
@@ -258,7 +258,7 @@ class Object {
static constexpr uint64_t kCombinedRefCountWeakOne =
details::kCombinedRefCountWeakOne;
static constexpr uint64_t kCombinedRefCountBothOne =
details::kCombinedRefCountBothOne;
/*! \brief increase strong reference count, the caller must already hold a
strong reference */
- void IncRef() {
+ TVM_FFI_INLINE void IncRef() {
#ifdef _MSC_VER
_InterlockedIncrement64(
reinterpret_cast<volatile __int64*>(&header_.combined_ref_count)); //
NOLINT(*)
@@ -313,7 +313,7 @@ class Object {
}
/*! \brief decrease strong reference count and delete the object */
- void DecRef() {
+ TVM_FFI_INLINE void DecRef() {
#ifdef _MSC_VER
// use simpler impl in windows to ensure correctness
uint64_t count_before_sub =
@@ -416,6 +416,8 @@ class Object {
template <typename T>
class ObjectPtr {
public:
+ // Core value members that perform ownership work are force-inlined so
refcount paths do not
+ // depend on compiler cost models. Defaulted and empty members intentionally
stay unmarked.
/*! \brief default constructor */
ObjectPtr() = default;
/*! \brief default constructor */
@@ -439,7 +441,7 @@ class ObjectPtr {
* \brief move constructor
* \param other The value to be moved
*/
- ObjectPtr(ObjectPtr<T>&& other) // NOLINT(*)
+ TVM_FFI_INLINE ObjectPtr(ObjectPtr<T>&& other) // NOLINT(*)
: data_(other.data_) {
other.data_ = nullptr;
}
@@ -448,13 +450,13 @@ class ObjectPtr {
* \param other The value to be moved
*/
template <typename Y>
- ObjectPtr(ObjectPtr<Y>&& other) // NOLINT(*)
+ TVM_FFI_INLINE ObjectPtr(ObjectPtr<Y>&& other) // NOLINT(*)
: data_(other.data_) {
static_assert(std::is_base_of_v<T, Y>, "can only assign of child class
ObjectPtr to parent");
other.data_ = nullptr;
}
/*! \brief destructor */
- ~ObjectPtr() { this->reset(); }
+ TVM_FFI_INLINE ~ObjectPtr() { this->reset(); }
/*!
* \brief Swap this array with another Object
* \param other The other Object
@@ -492,7 +494,7 @@ class ObjectPtr {
* \param other The value to be assigned.
* \return reference to self.
*/
- ObjectPtr<T>& operator=(ObjectPtr<T>&& other) { // NOLINT(*)
+ TVM_FFI_INLINE ObjectPtr<T>& operator=(ObjectPtr<T>&& other) { // NOLINT(*)
// copy-and-swap idiom
ObjectPtr(std::move(other)).swap(*this); // NOLINT(*)
return *this;
@@ -811,11 +813,13 @@ class ObjectRef {
/*! \brief copy constructor */
ObjectRef(const ObjectRef& other) = default;
/*! \brief move constructor */
- ObjectRef(ObjectRef&& other) noexcept : data_(std::move(other.data_)) {
other.data_ = nullptr; }
+ TVM_FFI_INLINE ObjectRef(ObjectRef&& other) noexcept :
data_(std::move(other.data_)) {
+ other.data_ = nullptr;
+ }
/*! \brief copy assignment */
ObjectRef& operator=(const ObjectRef& other) = default;
/*! \brief move assignment */
- ObjectRef& operator=(ObjectRef&& other) noexcept {
+ TVM_FFI_INLINE ObjectRef& operator=(ObjectRef&& other) noexcept {
data_ = std::move(other.data_);
other.data_ = nullptr;
return *this;
diff --git a/include/tvm/ffi/optional.h b/include/tvm/ffi/optional.h
index 8f46b991..2b8dbd15 100644
--- a/include/tvm/ffi/optional.h
+++ b/include/tvm/ffi/optional.h
@@ -86,13 +86,16 @@ class Optional<T,
// NOLINTBEGIN(google-explicit-constructor)
Optional(const Optional& other) = default;
Optional(Optional&& other) noexcept = default;
- Optional(std::optional<T> other) : data_(std::move(other)) {}
+ TVM_FFI_INLINE Optional(std::optional<T> other) : data_(std::move(other)) {}
Optional(std::nullopt_t) {}
- Optional(T other) : data_(std::move(other)) {}
+ TVM_FFI_INLINE Optional(T other) : data_(std::move(other)) {}
// NOLINTEND(google-explicit-constructor)
Optional& operator=(const Optional& other) = default;
- Optional& operator=(Optional&& other) noexcept = default;
+ TVM_FFI_INLINE Optional& operator=(Optional&& other) noexcept {
+ data_ = std::move(other.data_);
+ return *this;
+ }
TVM_FFI_INLINE Optional& operator=(T other) {
data_ = std::move(other);
@@ -185,11 +188,11 @@ class Optional<T,
/*! \brief move constructor. */
Optional(Optional&& other) noexcept = default;
/*! \brief construct from a value of type T (copy). */
- Optional(const T& value) : data_(value) {}
+ TVM_FFI_INLINE Optional(const T& value) : data_(value) {}
/*! \brief construct from a value of type T (move). */
- Optional(T&& value) : data_(std::move(value)) {}
+ TVM_FFI_INLINE Optional(T&& value) : data_(std::move(value)) {}
/*! \brief construct from a std::optional<T>. */
- Optional(std::optional<T> other) {
+ TVM_FFI_INLINE Optional(std::optional<T> other) {
if (other.has_value()) {
data_ = Any(*std::move(other));
}
@@ -199,7 +202,10 @@ class Optional<T,
/*! \brief copy assignment. */
Optional& operator=(const Optional& other) = default;
/*! \brief move assignment. */
- Optional& operator=(Optional&& other) noexcept = default;
+ TVM_FFI_INLINE Optional& operator=(Optional&& other) noexcept {
+ data_ = std::move(other.data_);
+ return *this;
+ }
TVM_FFI_INLINE Optional& operator=(T other) {
data_ = Any(std::move(other));
@@ -348,7 +354,7 @@ class Optional<T,
private:
friend struct TypeTraits<Optional<T>>;
// construct directly from an Any backing store.
- explicit Optional(Any data) : data_(std::move(data)) {}
+ TVM_FFI_INLINE explicit Optional(Any data) : data_(std::move(data)) {}
TVM_FFI_INLINE AnyView ToAnyView() const { return data_.operator AnyView(); }
TVM_FFI_INLINE Any MoveToAny() && { return std::move(data_); }
/*! \brief The underlying Any backing store, kTVMFFINone represents nullopt.
*/
@@ -375,16 +381,19 @@ class Optional<T,
std::enable_if_t<use_object_ref_optional_v<T>>> : public Objec
explicit Optional(UnsafeInit tag) : ObjectRef(tag) {}
Optional(std::nullopt_t) {}
Optional(std::nullptr_t) {}
- Optional(std::optional<T> other) {
+ TVM_FFI_INLINE Optional(std::optional<T> other) {
if (other.has_value()) {
*this = *std::move(other);
}
}
- Optional(T other) : ObjectRef(std::move(other)) {}
+ TVM_FFI_INLINE Optional(T other) : ObjectRef(std::move(other)) {}
// NOLINTEND(google-explicit-constructor)
Optional& operator=(const Optional&) = default;
- Optional& operator=(Optional&&) noexcept = default;
+ TVM_FFI_INLINE Optional& operator=(Optional&& other) noexcept {
+ ObjectRef::operator=(std::move(other));
+ return *this;
+ }
TVM_FFI_INLINE Optional& operator=(T other) {
ObjectRef::operator=(std::move(other));
@@ -548,16 +557,19 @@ class Optional<T,
std::enable_if_t<is_object_ptr_type_v<T> || is_arc_type_v<T>>>
Optional(Optional&&) noexcept = default;
Optional(std::nullopt_t) : StorageType(nullptr) {}
Optional(std::nullptr_t) : StorageType(nullptr) {}
- Optional(std::optional<T> other) {
+ TVM_FFI_INLINE Optional(std::optional<T> other) {
if (other.has_value()) {
static_cast<StorageType&>(*this) = StorageType(std::move(*other));
}
}
- Optional(T value) : StorageType(std::move(value)) {}
+ TVM_FFI_INLINE Optional(T value) : StorageType(std::move(value)) {}
// NOLINTEND(google-explicit-constructor)
Optional& operator=(const Optional&) = default;
- Optional& operator=(Optional&&) noexcept = default;
+ TVM_FFI_INLINE Optional& operator=(Optional&& other) noexcept {
+ static_cast<StorageType&>(*this) =
std::move(static_cast<StorageType&>(other));
+ return *this;
+ }
TVM_FFI_INLINE Optional& operator=(T value) {
static_cast<StorageType&>(*this) = StorageType(std::move(value));
