https://gcc.gnu.org/g:acfdad706d8acca6c4ed6ef3b63c2e02f1c47881
commit r17-603-gacfdad706d8acca6c4ed6ef3b63c2e02f1c47881 Author: Nathan Myers <[email protected]> Date: Fri Apr 3 23:55:39 2026 -0400 libstdc++: Use allocate_at_least in vector, string (P0401) [PR118030] Implement as much of allocator<>::allocate_at_least as possible relying solely on known alignment behavior of standard operator new. Use allocator_at_least in string and vector to maximize usage of actually allocated storage, as revealed by the allocator in use. For user-supplied allocators this may make a big difference. Nothing is changed in include/ext/malloc_allocator or others. They can be updated at leisure, piecemeal. libstdc++-v3/ChangeLog: PR libstdc++/118030 * config/abi/pre/gnu.ver: Expose string::_S_allocate_at_least, _M_create_plus symbols. * include/bits/alloc_traits.h: (allocate_at_least): Delegate in allocator_traits<allocator<_Tp>> specialization to allocator<_Tp>::allocate_at_least, unconditionally; annotate [[__gnu__::always_inline__]]. (allocate_at_least): Declare "= delete;" in allocator<void>. * include/bits/allocator.h (allocate_at_least): Delegate to base allocate_at_least where defined, calling with explicit base-class qualification, picking up __new_allocator member. * include/bits/basic_string.h: (_Alloc_result): Define new type. (_S_allocate_at_least): Define, using it. (_S_allocate): Minimize for legacy ABI use only. (_M_create_plus): Declare. (_M_create_and_place): Define, abstracting common operations. (assign): Use _S_allocate_at_least. * include/bits/basic_string.tcc: (_M_create_plus): Define. (_M_replace, reserve): Use _S_allocate_at_least. (_M_construct, others (3x)): Use _M_create_and_place. (_M_construct, input iterators): Use _M_create_plus. (_M_create, _M_assign, reserve, _M_mutate): Same. * include/bits/memory_resource.h (allocate_at_least): Define, document. * include/bits/new_allocator.h (allocate_at_least): Define. (_S_check_allocation_limit) Define. (allocate): Use _S_check_allocation_limit. (_S_max_size): Change from _M_max_size. (deallocate): Refine "if constexpr" logic. * include/bits/stl_vector.h: (_S_max_size): Move to _Vector_base. (_Alloc_result): Define type. (_M_allocate_at_least): Define, using allocate_at_least where supported. (_M_allocate): Delegate to _M_allocate_at_least. (max_size, _S_check_init_len): Use _S_max_size as moved. (_M_create_storage, append_range, _M_allocate_and_copy, _M_replace_storage): Define, abstracting common operations. (_M_replace_with): Define, likewise. (_M_range_initialize_n): Use _M_allocate_at_least. (_M_check_len): Improve logic. * include/bits/vector.tcc: (reserve, _M_fill_append, _M_range_insert): Use _M_allocate_at_least and _M_replace_storage. (operator=, _M_assign_aux): Use _M_replace_with. (_M_realloc_insert, _M_realloc_append, _M_default_append, insert_range): Use _M_allocate_at_least. (_M_fill_insert): Use _M_replace_storage, normalize whitespace. * testsuite/util/testsuite_allocator.h: (allocate_at_least (3x)): Define. (allocate): Use allocate_at_least. * testsuite/20_util/allocator/allocate_at_least.cc: Add tests. * testsuite/21_strings/basic_string/capacity/char/18654.cc: Loosen capacity check. * testsuite/21_strings/basic_string/capacity/char/shrink_to_fit.cc: Same. * testsuite/21_strings/basic_string/capacity/wchar_t/18654.cc: Same. * testsuite/21_strings/basic_string/capacity/wchar_t/2.cc: Same. * testsuite/21_strings/basic_string/capacity/wchar_t/shrink_to_fit.cc: Same. * testsuite/23_containers/vector/capacity/shrink_to_fit.cc: Same. * testsuite/23_containers/vector/capacity/shrink_to_fit2.cc: Same * testsuite/23_containers/vector/modifiers/emplace/self_emplace.cc: Adapt to looser reserve behavior. Diff: --- libstdc++-v3/config/abi/pre/gnu.ver | 4 + libstdc++-v3/include/bits/alloc_traits.h | 18 ++- libstdc++-v3/include/bits/allocator.h | 30 ++-- libstdc++-v3/include/bits/basic_string.h | 44 +++++- libstdc++-v3/include/bits/basic_string.tcc | 96 +++++++------ libstdc++-v3/include/bits/memory_resource.h | 17 +++ libstdc++-v3/include/bits/new_allocator.h | 77 +++++++--- libstdc++-v3/include/bits/stl_vector.h | 136 +++++++++++++----- libstdc++-v3/include/bits/vector.tcc | 155 ++++++++------------- .../20_util/allocator/allocate_at_least.cc | 108 +++++++++++++- .../21_strings/basic_string/capacity/char/18654.cc | 7 +- .../basic_string/capacity/char/shrink_to_fit.cc | 7 +- .../basic_string/capacity/wchar_t/18654.cc | 8 +- .../21_strings/basic_string/capacity/wchar_t/2.cc | 13 +- .../basic_string/capacity/wchar_t/shrink_to_fit.cc | 7 +- .../23_containers/vector/capacity/shrink_to_fit.cc | 7 +- .../vector/capacity/shrink_to_fit2.cc | 14 +- .../vector/modifiers/emplace/self_emplace.cc | 36 ++--- libstdc++-v3/testsuite/util/testsuite_allocator.h | 43 ++++++ 19 files changed, 586 insertions(+), 241 deletions(-) diff --git a/libstdc++-v3/config/abi/pre/gnu.ver b/libstdc++-v3/config/abi/pre/gnu.ver index 06892cd2d18b..bf4d7338cb6c 100644 --- a/libstdc++-v3/config/abi/pre/gnu.ver +++ b/libstdc++-v3/config/abi/pre/gnu.ver @@ -2628,6 +2628,10 @@ GLIBCXX_3.4.36 { _ZNSt6chrono8__detail25__recent_leap_second_infoERNS_16leap_second_infoEj; + # basic_string::allocate_at_least + _ZNSt7__cxx1112basic_stringI[cw]St11char_traitsI[cw]ESaI[cw]EE*_S_allocate_*; + _ZNSt7__cxx1112basic_stringI[cw]St11char_traitsI[cw]ESaI[cw]EE*_M_create_*; + } GLIBCXX_3.4.35; # Symbols in the support library (libsupc++) have their own tag. diff --git a/libstdc++-v3/include/bits/alloc_traits.h b/libstdc++-v3/include/bits/alloc_traits.h index e4e0f1608dec..101badff4549 100644 --- a/libstdc++-v3/include/bits/alloc_traits.h +++ b/libstdc++-v3/include/bits/alloc_traits.h @@ -417,9 +417,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION * is obliged to reserve more space than required for the cited * `n` objects, it may deliver the extra space to the caller. */ - [[nodiscard]] static constexpr auto + [[nodiscard]] static constexpr allocation_result<pointer, size_type> allocate_at_least(_Alloc& __a, size_type __n) - -> allocation_result<pointer, size_type> { if constexpr (requires { __a.allocate_at_least(__n); }) return __a.allocate_at_least(__n); @@ -664,14 +663,14 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION * @brief Allocate memory, generously. * @param __a An allocator. * @param __n The minimum number of objects to allocate space for. - * @return Memory of suitable size and alignment for `n` or more - * contiguous objects of type `value_type`. + * @return Memory of suitable size and alignment for `m >= n` + * contiguous objects of type `value_type`, and `m`. * * Returns `a.allocate_at_least(n)`. */ - [[nodiscard]] static constexpr auto + [[nodiscard,__gnu__::__always_inline__]] + static constexpr allocation_result<pointer, size_type> allocate_at_least(allocator_type& __a, size_type __n) - -> allocation_result<pointer, size_type> { return __a.allocate_at_least(__n); } #endif @@ -822,6 +821,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION static void* allocate(allocator_type&, size_type, const void* = nullptr) = delete; +#ifdef __glibcxx_allocate_at_least + static allocation_result<void*, size_type> + allocate_at_least(allocator_type&, size_type) = delete; +#endif + /// deallocate is ill-formed for allocator<void> static void deallocate(allocator_type&, void*, size_type) = delete; @@ -872,7 +876,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION select_on_container_copy_construction(const allocator_type& __rhs) { return __rhs; } }; -#endif +#endif // _GLIBCXX_HOSTED /// @cond undocumented #pragma GCC diagnostic push diff --git a/libstdc++-v3/include/bits/allocator.h b/libstdc++-v3/include/bits/allocator.h index 9c22c805ebe7..9c9ebdeb7961 100644 --- a/libstdc++-v3/include/bits/allocator.h +++ b/libstdc++-v3/include/bits/allocator.h @@ -193,9 +193,9 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION constexpr _Tp* allocate(size_t __n) { -#if __cpp_concepts +# if __cpp_concepts if constexpr (requires { sizeof(_Tp); }) -#endif +# endif if (std::__is_constant_evaluated()) { if (__builtin_mul_overflow(__n, sizeof(_Tp), &__n)) @@ -205,7 +205,27 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION return __allocator_base<_Tp>::allocate(__n, 0); } +#endif + +#ifdef __glibcxx_allocate_at_least // C++23 + [[nodiscard,__gnu__::__always_inline__]] + constexpr allocation_result<_Tp*, size_t> + allocate_at_least(size_t __n) + { + if consteval + { return { allocate(__n), __n }; } + else + { + if constexpr (requires + { __allocator_base<_Tp>::allocate_at_least(__n); }) + return __allocator_base<_Tp>::allocate_at_least(__n); + else + return { __allocator_base<_Tp>::allocate(__n), __n }; + } + } +#endif +#if __cpp_constexpr_dynamic_alloc // >= C++20 [[__gnu__::__always_inline__]] constexpr void deallocate(_Tp* __p, size_t __n) @@ -219,12 +239,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION } #endif // C++20 -#ifdef __glibcxx_allocate_at_least // C++23 - [[nodiscard]] constexpr allocation_result<_Tp*, size_t> - allocate_at_least(size_t __n) - { return { this->allocate(__n), __n }; } -#endif - friend __attribute__((__always_inline__)) _GLIBCXX20_CONSTEXPR bool operator==(const allocator&, const allocator&) _GLIBCXX_NOTHROW diff --git a/libstdc++-v3/include/bits/basic_string.h b/libstdc++-v3/include/bits/basic_string.h index af4e5d9486fb..c1871633f1b5 100644 --- a/libstdc++-v3/include/bits/basic_string.h +++ b/libstdc++-v3/include/bits/basic_string.h @@ -135,20 +135,35 @@ _GLIBCXX_BEGIN_NAMESPACE_CXX11 #endif private: + // For ABI reasons this must remain, though unused. static _GLIBCXX20_CONSTEXPR pointer _S_allocate(_Char_alloc_type& __a, size_type __n) + { return _Alloc_traits::allocate(__a, __n); } + + struct _Alloc_result { pointer __ptr; size_type __count; }; + + static _GLIBCXX20_CONSTEXPR _Alloc_result + _S_allocate_at_least(_Char_alloc_type& __a, size_type __n) { - pointer __p = _Alloc_traits::allocate(__a, __n); + _Alloc_result __r; +#ifdef __glibcxx_allocate_at_least // C++23 + auto [__ptr, __count] = _Alloc_traits::allocate_at_least(__a, __n); + __r.__ptr = __ptr; + __r.__count = __count; +#else + __r.__ptr = _Alloc_traits::allocate(__a, __n); + __r.__count = __n; +#endif #if __glibcxx_constexpr_string >= 201907L // std::char_traits begins the lifetime of characters, // but custom traits might not, so do it here. if constexpr (!is_same_v<_Traits, char_traits<_CharT>>) if (std::__is_constant_evaluated()) // Begin the lifetime of characters in allocated storage. - for (size_type __i = 0; __i < __n; ++__i) - std::construct_at(__builtin_addressof(__p[__i])); + for (size_type __i = 0; __i < __r.__count; ++__i) + std::construct_at(__builtin_addressof(__r.__ptr[__i])); #endif - return __p; + return __r; } #ifdef __glibcxx_string_view // >= C++17 @@ -287,6 +302,21 @@ _GLIBCXX_BEGIN_NAMESPACE_CXX11 // Create & Destroy _GLIBCXX20_CONSTEXPR + _Alloc_result + _M_create_plus(size_type __new_capacity, size_type __old_capacity); + + __attribute__((__always_inline__)) + _GLIBCXX20_CONSTEXPR + void + _M_create_and_place(size_type __new_capacity, size_type __old_capacity) + { + _Alloc_result __r = _M_create_plus(__new_capacity, __old_capacity); + _M_data(__r.__ptr); + _M_capacity(__r.__count - 1); // Leave room for NUL. + } + + // This must remain for ABI stability though unused. + _GLIBCXX20_CONSTEXPR pointer _M_create(size_type&, size_type); @@ -1782,10 +1812,10 @@ _GLIBCXX_BEGIN_NAMESPACE_CXX11 const auto __len = __str.size(); auto __alloc = __str._M_get_allocator(); // If this allocation throws there are no effects: - auto __ptr = _S_allocate(__alloc, __len + 1); + auto __r = _S_allocate_at_least(__alloc, __len + 1); _M_destroy(_M_allocated_capacity); - _M_data(__ptr); - _M_capacity(__len); + _M_data(__r.__ptr); + _M_capacity(__r.__count - 1); _M_set_length(__len); } } diff --git a/libstdc++-v3/include/bits/basic_string.tcc b/libstdc++-v3/include/bits/basic_string.tcc index b00dd550237b..5f83991db83f 100644 --- a/libstdc++-v3/include/bits/basic_string.tcc +++ b/libstdc++-v3/include/bits/basic_string.tcc @@ -139,14 +139,14 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION template<typename _CharT, typename _Traits, typename _Alloc> _GLIBCXX20_CONSTEXPR - typename basic_string<_CharT, _Traits, _Alloc>::pointer + typename basic_string<_CharT, _Traits, _Alloc>::_Alloc_result basic_string<_CharT, _Traits, _Alloc>:: - _M_create(size_type& __capacity, size_type __old_capacity) + _M_create_plus(size_type __capacity, size_type __old_capacity) { // _GLIBCXX_RESOLVE_LIB_DEFECTS // 83. String::npos vs. string::max_size() if (__capacity > max_size()) - std::__throw_length_error(__N("basic_string::_M_create")); + std::__throw_length_error(__N("basic_string::_M_create_plus")); // The below implements an exponential growth policy, necessary to // meet amortized linear time requirements of the library: see @@ -161,7 +161,19 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION // NB: Need an array of char_type[__capacity], plus a terminating // null char_type() element. - return _S_allocate(_M_get_allocator(), __capacity + 1); + return _S_allocate_at_least(_M_get_allocator(), __capacity + 1); + } + + // This must remain for ABI stability, though unused in current code. + template<typename _CharT, typename _Traits, typename _Alloc> + _GLIBCXX20_CONSTEXPR + typename basic_string<_CharT, _Traits, _Alloc>::pointer + basic_string<_CharT, _Traits, _Alloc>:: + _M_create(size_type& __capacity, size_type __old_capacity) + { + _Alloc_result __r = _M_create_plus(__capacity, __old_capacity); + __capacity = __r.__count - 1; // Leave room for NUL. + return __r.__ptr; } // NB: This is the special case for Input Iterators, used in @@ -203,11 +215,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION if (__len == __capacity) { // Allocate more space. - __capacity = __len + 1; - pointer __another = _M_create(__capacity, __len); - this->_S_copy(__another, _M_data(), __len); + _Alloc_result __another = _M_create_plus(__len + 1, __len); + __capacity = __another.__count - 1; // Leave room for NUL. + this->_S_copy(__another.__ptr, _M_data(), __len); _M_dispose(); - _M_data(__another); + _M_data(__another.__ptr); _M_capacity(__capacity); } traits_type::assign(_M_data()[__len++], @@ -231,10 +243,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION size_type __dnew = static_cast<size_type>(std::distance(__beg, __end)); if (__dnew > size_type(_S_local_capacity)) - { - _M_data(_M_create(__dnew, size_type(0))); - _M_capacity(__dnew); - } + _M_create_and_place(__dnew, size_type(0)); else _M_init_local_buf(); @@ -264,10 +273,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION _M_construct(size_type __n, _CharT __c) { if (__n > size_type(_S_local_capacity)) - { - _M_data(_M_create(__n, size_type(0))); - _M_capacity(__n); - } + _M_create_and_place(__n, size_type(0)); else _M_init_local_buf(); @@ -281,16 +287,13 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION // than difference between iterators. template<typename _CharT, typename _Traits, typename _Alloc> template<bool _Terminated> - _GLIBCXX20_CONSTEXPR + _GLIBCXX20_CONSTEXPR void basic_string<_CharT, _Traits, _Alloc>:: _M_construct(const _CharT* __str, size_type __n) { if (__n > size_type(_S_local_capacity)) - { - _M_data(_M_create(__n, size_type(0))); - _M_capacity(__n); - } + _M_create_and_place(__n, size_type(0)); else _M_init_local_buf(); @@ -347,11 +350,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION if (__rsize > __capacity) { - size_type __new_capacity = __rsize; - pointer __tmp = _M_create(__new_capacity, __capacity); + // if _M_create_plus throws, there is no effect. + _Alloc_result __tmp = _M_create_plus(__rsize, __capacity); _M_dispose(); - _M_data(__tmp); - _M_capacity(__new_capacity); + _M_data(__tmp.__ptr); + _M_capacity(__tmp.__count - 1); } if (__rsize) @@ -375,11 +378,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION if (__res <= __capacity) return; - pointer __tmp = _M_create(__res, __capacity); - this->_S_copy(__tmp, _M_data(), length() + 1); + _Alloc_result __r = _M_create_plus(__res, __capacity); + this->_S_copy(__r.__ptr, _M_data(), length() + 1); _M_dispose(); - _M_data(__tmp); - _M_capacity(__res); + _M_data(__r.__ptr); + _M_capacity(__r.__count - 1); // Leave room for NUL. } template<typename _CharT, typename _Traits, typename _Alloc> @@ -392,19 +395,19 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION const size_type __how_much = length() - __pos - __len1; size_type __new_capacity = length() + __len2 - __len1; - pointer __r = _M_create(__new_capacity, capacity()); + _Alloc_result __r = _M_create_plus(__new_capacity, capacity()); if (__pos) - this->_S_copy(__r, _M_data(), __pos); + this->_S_copy(__r.__ptr, _M_data(), __pos); if (__s && __len2) - this->_S_copy(__r + __pos, __s, __len2); + this->_S_copy(__r.__ptr + __pos, __s, __len2); if (__how_much) - this->_S_copy(__r + __pos + __len2, + this->_S_copy(__r.__ptr + __pos + __len2, _M_data() + __pos + __len1, __how_much); _M_dispose(); - _M_data(__r); - _M_capacity(__new_capacity); + _M_data(__r.__ptr); + _M_capacity(__r.__count - 1); // Leave room for NUL. } template<typename _CharT, typename _Traits, typename _Alloc> @@ -430,6 +433,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION if (_M_is_local()) return; +#ifdef __glibcxx_allocate_at_least // C++23 + const size_type __limit = (__STDCPP_DEFAULT_NEW_ALIGNMENT__ - 1) / sizeof(_CharT); +#else + const size_type __limit = 0; +#endif const size_type __length = length(); const size_type __capacity = _M_allocated_capacity; @@ -441,14 +449,15 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION _M_data(_M_local_data()); } #if __cpp_exceptions - else if (__length < __capacity) + else if (__capacity - __length > __limit ) try { - pointer __tmp = _S_allocate(_M_get_allocator(), __length + 1); - this->_S_copy(__tmp, _M_data(), __length + 1); + _Alloc_result __r = _S_allocate_at_least( + _M_get_allocator(), __length + 1); + this->_S_copy(__r.__ptr, _M_data(), __length + 1); _M_dispose(); - _M_data(__tmp); - _M_capacity(__length); + _M_data(__r.__ptr); + _M_capacity(__r.__count - 1); // reserve room for NUL. } catch (const __cxxabiv1::__forced_unwind&) { throw; } @@ -588,7 +597,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION #if __cpp_lib_is_constant_evaluated if (std::is_constant_evaluated()) { - auto __newp = _S_allocate(_M_get_allocator(), __new_size); + auto __newp = + _S_allocate_at_least(_M_get_allocator(), __new_size).__ptr; _S_copy(__newp, this->_M_data(), __pos); _S_copy(__newp + __pos, __s, __len2); _S_copy(__newp + __pos + __len2, __p + __len1, __how_much); @@ -677,7 +687,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION #endif // C++11 #endif // _GLIBCXX_USE_CXX11_ABI - + #if __glibcxx_constexpr_string >= 201907L # define _GLIBCXX_STRING_CONSTEXPR constexpr #else @@ -916,7 +926,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION // Avoid reallocation for common case. __str.erase(); _CharT __buf[128]; - __size_type __len = 0; + __size_type __len = 0; const streamsize __w = __in.width(); const __size_type __n = __w > 0 ? static_cast<__size_type>(__w) : __str.max_size(); diff --git a/libstdc++-v3/include/bits/memory_resource.h b/libstdc++-v3/include/bits/memory_resource.h index e5c6697b07e3..a9db58b76a34 100644 --- a/libstdc++-v3/include/bits/memory_resource.h +++ b/libstdc++-v3/include/bits/memory_resource.h @@ -48,6 +48,7 @@ # include <bits/utility.h> // index_sequence # include <tuple> // tuple, forward_as_tuple #endif +#include <bits/memoryfwd.h> namespace std _GLIBCXX_VISIBILITY(default) { @@ -468,6 +469,22 @@ namespace pmr allocate(allocator_type& __a, size_type __n, const_void_pointer) { return __a.allocate(__n); } +#ifdef __glibcxx_allocate_at_least + /** + * @brief Allocate memory. + * @param __a An allocator. + * @param __n The number of objects to allocate space for. + * @return Memory of suitable size and alignment for `n` objects + * of type `value_type`. + * + * Just returns `{ a.allocate(n), n }`: `polymorphic_allocator` + * cannot be extended without breaking ABI. + */ + [[nodiscard]] static std::allocation_result<pointer, size_type> + allocate_at_least(allocator_type& __a, size_type __n) + { return { __a.allocate(__n), __n }; } +#endif + /** * @brief Deallocate memory. * @param __a An allocator. diff --git a/libstdc++-v3/include/bits/new_allocator.h b/libstdc++-v3/include/bits/new_allocator.h index fbe03e392aae..4524355a4a0f 100644 --- a/libstdc++-v3/include/bits/new_allocator.h +++ b/libstdc++-v3/include/bits/new_allocator.h @@ -37,6 +37,7 @@ #if __cplusplus >= 201103L #include <type_traits> #endif +#include <bits/memoryfwd.h> namespace std _GLIBCXX_VISIBILITY(default) { @@ -123,6 +124,20 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wc++17-extensions" // if constexpr + __attribute__((__always_inline__)) + _GLIBCXX20_CONSTEXPR static void + _S_check_allocation_limit(size_t __n) + { + if (__builtin_expect(__n > _S_max_size(), false)) + { + // _GLIBCXX_RESOLVE_LIB_DEFECTS + // 3190. allocator::allocate sometimes returns too little storage + if (__n > (std::size_t(-1) / sizeof(_Tp))) + std::__throw_bad_array_new_length(); + std::__throw_bad_alloc(); + } + } + // NB: __n is permitted to be 0. The C++ standard says nothing // about what the return value is when __n == 0. _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR _Tp* @@ -142,25 +157,49 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION else #endif #endif - if (__builtin_expect(__n > this->_M_max_size(), false)) { - // _GLIBCXX_RESOLVE_LIB_DEFECTS - // 3190. allocator::allocate sometimes returns too little storage - if (__n > (std::size_t(-1) / sizeof(_Tp))) - std::__throw_bad_array_new_length(); - std::__throw_bad_alloc(); - } + _S_check_allocation_limit(__n); #if __cpp_aligned_new && __cplusplus >= 201103L - else if constexpr (alignof(_Tp) > __STDCPP_DEFAULT_NEW_ALIGNMENT__) + if constexpr (alignof(_Tp) > __STDCPP_DEFAULT_NEW_ALIGNMENT__) + { + std::align_val_t __al = std::align_val_t(alignof(_Tp)); + return static_cast<_Tp*>( + _GLIBCXX_OPERATOR_NEW(__n * sizeof(_Tp), __al)); + } + else +#endif + return static_cast<_Tp*>( + _GLIBCXX_OPERATOR_NEW(__n * sizeof(_Tp))); + } + } + +#ifdef __glibcxx_allocate_at_least // C++23 + [[nodiscard]] constexpr std::allocation_result<_Tp*, size_t> + allocate_at_least(size_t __n) + { + if ! consteval { - std::align_val_t __al = std::align_val_t(alignof(_Tp)); - return static_cast<_Tp*>(_GLIBCXX_OPERATOR_NEW(__n * sizeof(_Tp), - __al)); + if constexpr (requires { sizeof(_Tp); }) + if constexpr (alignof(_Tp) <= __STDCPP_DEFAULT_NEW_ALIGNMENT__) + if constexpr ( sizeof(_Tp) < __STDCPP_DEFAULT_NEW_ALIGNMENT__) + { + _S_check_allocation_limit(__n); + const size_t __need = __n * sizeof(_Tp); + const size_t __mask = __STDCPP_DEFAULT_NEW_ALIGNMENT__ - 1; + size_t __ask = (__need + __mask) & ~__mask; + // Avoid rounding up to and asking for 2^63 bytes (PR108377): + __ask -= __ask >> (__SIZE_WIDTH__ - 1); + auto* __p = static_cast<_Tp*>(_GLIBCXX_OPERATOR_NEW(__ask)); + using _U8 = const unsigned char; + static_assert(sizeof(_Tp) <= ~_U8()); + // Use 8-bit division for minimal latency: + _U8 __spare = __ask - __need, __size = sizeof(_Tp); + return { __p , __n + __spare / __size }; + } } -#endif - else - return static_cast<_Tp*>(_GLIBCXX_OPERATOR_NEW(__n * sizeof(_Tp))); + return { allocate(__n), __n }; } +#endif // __p is not permitted to be a null pointer. _GLIBCXX20_CONSTEXPR void @@ -177,10 +216,10 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION { _GLIBCXX_OPERATOR_DELETE(_GLIBCXX_SIZED_DEALLOC(__p, __n), std::align_val_t(alignof(_Tp))); - return; } + else #endif - _GLIBCXX_OPERATOR_DELETE(_GLIBCXX_SIZED_DEALLOC(__p, __n)); + _GLIBCXX_OPERATOR_DELETE(_GLIBCXX_SIZED_DEALLOC(__p, __n)); } #pragma GCC diagnostic pop @@ -192,7 +231,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION __attribute__((__always_inline__)) size_type max_size() const _GLIBCXX_USE_NOEXCEPT - { return _M_max_size(); } + { return _S_max_size(); } #if __cplusplus >= 201103L template<typename _Up, typename... _Args> @@ -238,8 +277,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION private: __attribute__((__always_inline__)) - _GLIBCXX_CONSTEXPR size_type - _M_max_size() const _GLIBCXX_USE_NOEXCEPT + _GLIBCXX_CONSTEXPR static size_type + _S_max_size() _GLIBCXX_USE_NOEXCEPT { #if __PTRDIFF_MAX__ < __SIZE_MAX__ return std::size_t(__PTRDIFF_MAX__) / sizeof(_Tp); diff --git a/libstdc++-v3/include/bits/stl_vector.h b/libstdc++-v3/include/bits/stl_vector.h index c4ca214752a0..0c9b74fdbb24 100644 --- a/libstdc++-v3/include/bits/stl_vector.h +++ b/libstdc++-v3/include/bits/stl_vector.h @@ -317,6 +317,19 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER get_allocator() const _GLIBCXX_NOEXCEPT { return allocator_type(_M_get_Tp_allocator()); } + static _GLIBCXX20_CONSTEXPR size_t + _S_max_size(const _Tp_alloc_type& __a) _GLIBCXX_NOEXCEPT + { + // std::distance(begin(), end()) cannot be greater than PTRDIFF_MAX, + // and realistically we can't store more than PTRDIFF_MAX/sizeof(T) + // (even if std::allocator_traits::max_size says we can). + const size_t __diffmax = + __gnu_cxx::__numeric_traits<ptrdiff_t>::__max / sizeof(_Tp); + const size_t __allocmax = + __gnu_cxx::__alloc_traits<_Alloc>::max_size(__a); + return (std::min)(__diffmax, __allocmax); + } + #if __cplusplus >= 201103L _Vector_base() = default; #else @@ -389,6 +402,43 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER return __n != 0 ? _Tr::allocate(_M_impl, __n) : pointer(); } + struct _Alloc_result { pointer __ptr; size_t __count; }; + + _GLIBCXX20_CONSTEXPR + _Alloc_result + _M_allocate_at_least(size_t __n) + { + typedef __gnu_cxx::__alloc_traits<_Tp_alloc_type> _Tr; + _Alloc_result __r; + if (__builtin_expect(__n != 0, true)) + { +#ifdef __glibcxx_allocate_at_least // C++23 + if constexpr (requires { _Tr::allocate_at_least(_M_impl, __n); }) + { + auto [__ptr, __count] = _Tr::allocate_at_least(_M_impl, __n); + if (__count > __n) + { + size_t __max = _S_max_size(_M_get_Tp_allocator()); + if (__builtin_expect(__count > __max, false)) + __count = __max; + } + __r = { __ptr, __count }; + } + else +#endif + { + __r.__ptr = _Tr::allocate(_M_impl, __n); + __r.__count = __n; + } + } + else + { + __r.__ptr = pointer(); + __r.__count = 0; + } + return __r; + } + _GLIBCXX20_CONSTEXPR void _M_deallocate(pointer __p, size_t __n) @@ -404,9 +454,9 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER void _M_create_storage(size_t __n) { - this->_M_impl._M_start = this->_M_allocate(__n); - this->_M_impl._M_finish = this->_M_impl._M_start; - this->_M_impl._M_end_of_storage = this->_M_impl._M_start + __n; + _Alloc_result __r = this->_M_allocate_at_least(__n); + this->_M_impl._M_finish = this->_M_impl._M_start = __r.__ptr; + this->_M_impl._M_end_of_storage = this->_M_impl._M_start + __r.__count; } #if __glibcxx_containers_ranges // C++ >= 23 @@ -480,6 +530,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER typedef _Vector_base<_Tp, _Alloc> _Base; typedef typename _Base::_Tp_alloc_type _Tp_alloc_type; typedef __gnu_cxx::__alloc_traits<_Tp_alloc_type> _Alloc_traits; + typedef typename _Base::_Alloc_result _Alloc_result; public: typedef _Tp value_type; @@ -536,6 +587,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER protected: using _Base::_M_allocate; + using _Base::_M_allocate_at_least; using _Base::_M_deallocate; using _Base::_M_impl; using _Base::_M_get_Tp_allocator; @@ -1116,7 +1168,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR size_type max_size() const _GLIBCXX_NOEXCEPT - { return _S_max_size(_M_get_Tp_allocator()); } + { return _Base::_S_max_size(_M_get_Tp_allocator()); } #if __cplusplus >= 201103L /** @@ -1682,16 +1734,17 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER return; } - const size_type __len = _M_check_len(__n, "vector::append_range"); + const size_type __ask = _M_check_len(__n, "vector::append_range"); pointer __old_start = this->_M_impl._M_start; pointer __old_finish = this->_M_impl._M_finish; - allocator_type& __a = _M_get_Tp_allocator(); - const pointer __start = this->_M_allocate(__len); + auto [__ptr, __got] = this->_M_allocate_at_least(__ask); + const pointer __start = __ptr; const pointer __mid = __start + __sz; const pointer __back = __mid + __n; - _Guard_alloc __guard(__start, __len, *this); + _Guard_alloc __guard(__start, __got, *this); + allocator_type& __a = _M_get_Tp_allocator(); std::__uninitialized_copy_a(ranges::begin(__rg), ranges::end(__rg), __mid, __a); @@ -1733,7 +1786,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER // Finally, take ownership of new storage: this->_M_impl._M_start = __start; this->_M_impl._M_finish = __back; - this->_M_impl._M_end_of_storage = __start + __len; + this->_M_impl._M_end_of_storage = __start + __got; } else { @@ -1889,20 +1942,45 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER protected: /** * Memory expansion handler. Uses the member allocation function to - * obtain @a n bytes of memory, and then copies [first,last) into it. + * obtain at least `n` objects worth of memory, copies `[first,last)` + * into it, reports what was actually allocated. */ template<typename _ForwardIterator> _GLIBCXX20_CONSTEXPR - pointer + _Alloc_result _M_allocate_and_copy(size_type __n, _ForwardIterator __first, _ForwardIterator __last) { - _Guard_alloc __guard(this->_M_allocate(__n), __n, *this); + _Alloc_result __r = this->_M_allocate_at_least(__n); + _Guard_alloc __guard(__r.__ptr, __r.__count, *this); std::__uninitialized_copy_a (__first, __last, __guard._M_storage, _M_get_Tp_allocator()); - return __guard._M_release(); + (void) __guard._M_release(); + return __r; } + _GLIBCXX20_CONSTEXPR void + _M_replace_storage(pointer __start, pointer __end, size_type __cap) + { + _GLIBCXX_ASAN_ANNOTATE_REINIT; + _M_deallocate(this->_M_impl._M_start, + this->_M_impl._M_end_of_storage - this->_M_impl._M_start); + this->_M_impl._M_start = __start; + this->_M_impl._M_finish = __end; + this->_M_impl._M_end_of_storage = __start + __cap; + } + + template<typename _ForwardIterator> + _GLIBCXX20_CONSTEXPR + void + _M_replace_with(size_type __n, + _ForwardIterator __first, _ForwardIterator __last) + { + _Alloc_result __r = _M_allocate_and_copy(__n, __first, __last); + std::_Destroy(this->_M_impl._M_start, + this->_M_impl._M_finish, _M_get_Tp_allocator()); + _M_replace_storage(__r.__ptr, __r.__ptr + __n, __r.__count); + } // Internal constructor functions follow. @@ -1916,6 +1994,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER _M_initialize_dispatch(_Integer __int_n, _Integer __value, __true_type) { const size_type __n = static_cast<size_type>(__int_n); + // NB this is c++98 code, so has no allocate_at_least. pointer __start = _M_allocate(_S_check_init_len(__n, _M_get_Tp_allocator())); this->_M_impl._M_start = __start; @@ -1971,10 +2050,11 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER _M_range_initialize_n(_Iterator __first, _Sentinel __last, size_type __n) { - pointer __start = - this->_M_allocate(_S_check_init_len(__n, _M_get_Tp_allocator())); + _Alloc_result __r = this->_M_allocate_at_least( + _S_check_init_len(__n, _M_get_Tp_allocator())); + pointer __start = __r.__ptr; this->_M_impl._M_start = this->_M_impl._M_finish = __start; - this->_M_impl._M_end_of_storage = __start + __n; + this->_M_impl._M_end_of_storage = __start + __r.__count; this->_M_impl._M_finish = std::__uninitialized_copy_a(_GLIBCXX_MOVE(__first), __last, __start, _M_get_Tp_allocator()); @@ -2191,35 +2271,27 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER size_type _M_check_len(size_type __n, const char* __s) const { - if (max_size() - size() < __n) + const size_type __room = max_size() - size(); + if (__room < __n) __throw_length_error(__N(__s)); - const size_type __len = size() + (std::max)(size(), __n); - return (__len < size() || __len > max_size()) ? max_size() : __len; + if (__n < size()) + __n = size(); // Grow by (at least) doubling ... + if (__n > __room) + __n = __room; // ... but only as much as will fit. + return size() + __n; } // Called by constructors to check initial size. static _GLIBCXX20_CONSTEXPR size_type _S_check_init_len(size_type __n, const allocator_type& __a) { - if (__n > _S_max_size(_Tp_alloc_type(__a))) + if (__n > _Base::_S_max_size(_Tp_alloc_type(__a))) __throw_length_error( __N("cannot create std::vector larger than max_size()")); return __n; } - static _GLIBCXX20_CONSTEXPR size_type - _S_max_size(const _Tp_alloc_type& __a) _GLIBCXX_NOEXCEPT - { - // std::distance(begin(), end()) cannot be greater than PTRDIFF_MAX, - // and realistically we can't store more than PTRDIFF_MAX/sizeof(T) - // (even if std::allocator_traits::max_size says we can). - const size_t __diffmax - = __gnu_cxx::__numeric_traits<ptrdiff_t>::__max / sizeof(_Tp); - const size_t __allocmax = _Alloc_traits::max_size(__a); - return (std::min)(__diffmax, __allocmax); - } - // Internal erase functions follow. // Called by erase(q1,q2), clear(), resize(), _M_fill_assign, diff --git a/libstdc++-v3/include/bits/vector.tcc b/libstdc++-v3/include/bits/vector.tcc index b790fca2964e..86c41d74ff44 100644 --- a/libstdc++-v3/include/bits/vector.tcc +++ b/libstdc++-v3/include/bits/vector.tcc @@ -75,13 +75,13 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER if (this->capacity() < __n) { const size_type __old_size = size(); - pointer __tmp; + _Alloc_result __tmp; #if __cplusplus >= 201103L if constexpr (_S_use_relocate()) { - __tmp = this->_M_allocate(__n); + __tmp = this->_M_allocate_at_least(__n); std::__relocate_a(this->_M_impl._M_start, this->_M_impl._M_finish, - __tmp, _M_get_Tp_allocator()); + __tmp.__ptr, _M_get_Tp_allocator()); } else #endif @@ -92,13 +92,8 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER std::_Destroy(this->_M_impl._M_start, this->_M_impl._M_finish, _M_get_Tp_allocator()); } - _GLIBCXX_ASAN_ANNOTATE_REINIT; - _M_deallocate(this->_M_impl._M_start, - this->_M_impl._M_end_of_storage - - this->_M_impl._M_start); - this->_M_impl._M_start = __tmp; - this->_M_impl._M_finish = __tmp + __old_size; - this->_M_impl._M_end_of_storage = this->_M_impl._M_start + __n; + _M_replace_storage( + __tmp.__ptr, __tmp.__ptr + __old_size, __tmp.__count); } } #pragma GCC diagnostic pop @@ -106,12 +101,12 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER #if __cplusplus >= 201103L template<typename _Tp, typename _Alloc> template<typename... _Args> -#if __cplusplus > 201402L +# if __cplusplus > 201402L _GLIBCXX20_CONSTEXPR typename vector<_Tp, _Alloc>::reference -#else +# else void -#endif +# endif vector<_Tp, _Alloc>:: emplace_back(_Args&&... __args) { @@ -125,11 +120,11 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER } else _M_realloc_append(std::forward<_Args>(__args)...); -#if __cplusplus > 201402L +# if __cplusplus > 201402L return back(); -#endif +# endif } -#endif +#endif // __cplusplus >= 201103L template<typename _Tp, typename _Alloc> _GLIBCXX20_CONSTEXPR @@ -238,22 +233,10 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER #endif const size_type __xlen = __x.size(); if (__xlen > capacity()) - { - pointer __tmp = _M_allocate_and_copy(__xlen, __x.begin(), - __x.end()); - std::_Destroy(this->_M_impl._M_start, this->_M_impl._M_finish, - _M_get_Tp_allocator()); - _M_deallocate(this->_M_impl._M_start, - this->_M_impl._M_end_of_storage - - this->_M_impl._M_start); - this->_M_impl._M_start = __tmp; - this->_M_impl._M_end_of_storage = this->_M_impl._M_start + __xlen; - } + _M_replace_with(__xlen, __x.begin(), __x.end()); else if (size() >= __xlen) - { - std::_Destroy(std::copy(__x.begin(), __x.end(), begin()), - end(), _M_get_Tp_allocator()); - } + std::_Destroy(std::copy(__x.begin(), __x.end(), begin()), + end(), _M_get_Tp_allocator()); else { std::copy(__x._M_impl._M_start, __x._M_impl._M_start + size(), @@ -332,16 +315,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER __builtin_unreachable(); _S_check_init_len(__len, _M_get_Tp_allocator()); - pointer __tmp(_M_allocate_and_copy(__len, __first, __last)); - std::_Destroy(this->_M_impl._M_start, this->_M_impl._M_finish, - _M_get_Tp_allocator()); - _GLIBCXX_ASAN_ANNOTATE_REINIT; - _M_deallocate(this->_M_impl._M_start, - this->_M_impl._M_end_of_storage - - this->_M_impl._M_start); - this->_M_impl._M_start = __tmp; - this->_M_impl._M_finish = this->_M_impl._M_start + __len; - this->_M_impl._M_end_of_storage = this->_M_impl._M_finish; + _M_replace_with(__len, __first, __last); } else if (__sz >= __len) _M_erase_at_end(std::copy(__first, __last, this->_M_impl._M_start)); @@ -464,13 +438,15 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER _M_realloc_insert(iterator __position, const _Tp& __x) #endif { - const size_type __len = _M_check_len(1u, "vector::_M_realloc_insert"); - if (__len <= 0) + const size_type __len1 = _M_check_len(1u, "vector::_M_realloc_insert"); + if (__len1 <= 0) __builtin_unreachable(); pointer __old_start = this->_M_impl._M_start; pointer __old_finish = this->_M_impl._M_finish; const size_type __elems_before = __position - begin(); - pointer __new_start(this->_M_allocate(__len)); + _Alloc_result __r = this->_M_allocate_at_least(__len1); + const size_type __len = __r.__count; + pointer __new_start(__r.__ptr); pointer __new_finish(__new_start); { @@ -574,14 +550,16 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER const size_type __len = _M_check_len(1u, "vector::_M_realloc_append"); if (__len <= 0) __builtin_unreachable(); - pointer __old_start = this->_M_impl._M_start; - pointer __old_finish = this->_M_impl._M_finish; + const pointer __old_start = this->_M_impl._M_start; + const pointer __old_finish = this->_M_impl._M_finish; const size_type __elems = size(); - pointer __new_start(this->_M_allocate(__len)); + const _Alloc_result __r = this->_M_allocate_at_least(__len); + const size_type __rlen = __r.__count; + const pointer __new_start(__r.__ptr); pointer __new_finish(__new_start); { - _Guard_alloc __guard(__new_start, __len, *this); + _Guard_alloc __guard(__new_start, __rlen, *this); // The order of the three operations is dictated by the C++11 // case, where the moves could alter a new element belonging @@ -652,7 +630,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER this->_M_impl._M_start = __new_start; this->_M_impl._M_finish = __new_finish; - this->_M_impl._M_end_of_storage = __new_start + __len; + this->_M_impl._M_end_of_storage = __new_start + __rlen; } #pragma GCC diagnostic pop @@ -716,10 +694,12 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER pointer __old_finish = this->_M_impl._M_finish; const pointer __pos = __position.base(); - const size_type __len = + const size_type __len1 = _M_check_len(__n, "vector::_M_fill_insert"); const size_type __elems_before = __pos - __old_start; - pointer __new_start(this->_M_allocate(__len)); + _Alloc_result __r = this->_M_allocate_at_least(__len1); + const size_type __len = __r.__count; + pointer __new_start(__r.__ptr); pointer __new_finish(__new_start); __try { @@ -727,21 +707,16 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER std::__uninitialized_fill_n_a(__new_start + __elems_before, __n, __x, _M_get_Tp_allocator()); - __new_finish = pointer(); - - __new_finish - = std::__uninitialized_move_if_noexcept_a - (__old_start, __pos, __new_start, _M_get_Tp_allocator()); - + __new_finish = pointer(); // ... in case of a throw. + __new_finish = std::__uninitialized_move_if_noexcept_a( + __old_start, __pos, __new_start, _M_get_Tp_allocator()); __new_finish += __n; - - __new_finish - = std::__uninitialized_move_if_noexcept_a - (__pos, __old_finish, __new_finish, _M_get_Tp_allocator()); + __new_finish = std::__uninitialized_move_if_noexcept_a( + __pos, __old_finish, __new_finish, _M_get_Tp_allocator()); } __catch(...) { - if (!__new_finish) + if (__new_finish == pointer()) std::_Destroy(__new_start + __elems_before, __new_start + __elems_before + __n, _M_get_Tp_allocator()); @@ -752,12 +727,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER __throw_exception_again; } std::_Destroy(__old_start, __old_finish, _M_get_Tp_allocator()); - _GLIBCXX_ASAN_ANNOTATE_REINIT; - _M_deallocate(__old_start, - this->_M_impl._M_end_of_storage - __old_start); - this->_M_impl._M_start = __new_start; - this->_M_impl._M_finish = __new_finish; - this->_M_impl._M_end_of_storage = __new_start + __len; + _M_replace_storage(__new_start, __new_finish, __len); } } } @@ -785,9 +755,10 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER pointer __old_finish = this->_M_impl._M_finish; const size_type __old_size = __old_finish - __old_start; - const size_type __len = - _M_check_len(__n, "vector::_M_fill_append"); - pointer __new_start(this->_M_allocate(__len)); + size_type __len = _M_check_len(__n, "vector::_M_fill_append"); + _Alloc_result __r = this->_M_allocate_at_least(__len); + __len = __r.__count; + pointer __new_start(__r.__ptr); pointer __new_finish(__new_start + __old_size); __try { @@ -807,12 +778,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER __throw_exception_again; } std::_Destroy(__old_start, __old_finish, _M_get_Tp_allocator()); - _GLIBCXX_ASAN_ANNOTATE_REINIT; - _M_deallocate(__old_start, - this->_M_impl._M_end_of_storage - __old_start); - this->_M_impl._M_start = __new_start; - this->_M_impl._M_finish = __new_finish; - this->_M_impl._M_end_of_storage = __new_start + __len; + _M_replace_storage(__new_start, __new_finish, __len); } } @@ -852,9 +818,11 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER pointer __old_start = this->_M_impl._M_start; pointer __old_finish = this->_M_impl._M_finish; - const size_type __len = + const size_type __len1 = _M_check_len(__n, "vector::_M_default_append"); - pointer __new_start(this->_M_allocate(__len)); + _Alloc_result __r = this->_M_allocate_at_least(__len1); + const size_type __len = __r.__count; + pointer __new_start(__r.__ptr); { _Guard_alloc __guard(__new_start, __len, *this); @@ -1003,14 +971,16 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER pointer __old_start = this->_M_impl._M_start; pointer __old_finish = this->_M_impl._M_finish; - const size_type __len = + const size_type __ask = _M_check_len(__n, "vector::_M_range_insert"); #if __cplusplus < 201103L - if (__len < (__n + (__old_finish - __old_start))) + if (__ask < (__n + (__old_finish - __old_start))) __builtin_unreachable(); #endif - pointer __new_start(this->_M_allocate(__len)); + _Alloc_result __r = this->_M_allocate_at_least(__ask); + const size_type __got = __r.__count; + pointer __new_start(__r.__ptr); pointer __new_finish(__new_start); __try { @@ -1031,17 +1001,12 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER { std::_Destroy(__new_start, __new_finish, _M_get_Tp_allocator()); - _M_deallocate(__new_start, __len); + _M_deallocate(__new_start, __got); __throw_exception_again; } std::_Destroy(__old_start, __old_finish, _M_get_Tp_allocator()); - _GLIBCXX_ASAN_ANNOTATE_REINIT; - _M_deallocate(__old_start, - this->_M_impl._M_end_of_storage - __old_start); - this->_M_impl._M_start = __new_start; - this->_M_impl._M_finish = __new_finish; - this->_M_impl._M_end_of_storage = __new_start + __len; + _M_replace_storage(__new_start, __new_finish, __got); } } } @@ -1111,7 +1076,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER } else // Reallocate { - const size_type __len + const size_type __ask = _M_check_len(__n, "vector::insert_range"); struct _Guard : _Guard_alloc @@ -1130,8 +1095,10 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER }; // Allocate new storage: - pointer __new_start(this->_M_allocate(__len)); - _Guard __guard(__new_start, __len, *this); + _Alloc_result __r = this->_M_allocate_at_least(__ask); + const size_type __got = __r.__count; + pointer __new_start(__r.__ptr); + _Guard __guard(__new_start, __got, *this); auto& __alloc = _M_get_Tp_allocator(); @@ -1158,13 +1125,13 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER // with __guard so that it cleans up the old storage: this->_M_impl._M_start = __guard._M_storage; this->_M_impl._M_finish = __guard._M_finish; - this->_M_impl._M_end_of_storage = __new_start + __len; + this->_M_impl._M_end_of_storage = __new_start + __got; __guard._M_storage = __old_start; __guard._M_finish = __old_finish; __guard._M_len = (__old_finish - __old_start) + __cap; // _Asan::_Reinit destructor marks unused capacity. // _Guard destructor destroys [old_start,old_finish). - // _Guard_alloc destructor frees [old_start,old_start+len). + // _Guard_alloc destructor frees [old_start,old_start+got). } return begin() + __ins_idx; } diff --git a/libstdc++-v3/testsuite/20_util/allocator/allocate_at_least.cc b/libstdc++-v3/testsuite/20_util/allocator/allocate_at_least.cc index 5399096d294b..987e2472490a 100644 --- a/libstdc++-v3/testsuite/20_util/allocator/allocate_at_least.cc +++ b/libstdc++-v3/testsuite/20_util/allocator/allocate_at_least.cc @@ -29,7 +29,7 @@ template <typename T> } }; -int main() +void base() { std::allocator<X> native; auto a1 = native.allocate_at_least(100); @@ -63,3 +63,109 @@ int main() VERIFY(a5.ptr == minimal.keep); minimal_traits::deallocate(minimal, a5.ptr, a5.count); } + +void extra() +{ + using SatC = std::allocator_traits<std::allocator<char>>; + std::allocator<char> satc; + { + auto [p, n] = SatC::allocate_at_least(satc, 1); + VERIFY(n == __STDCPP_DEFAULT_NEW_ALIGNMENT__); + SatC::deallocate(satc, p, n); + } + { + auto [p, n] = SatC::allocate_at_least(satc, 2); + VERIFY(n == __STDCPP_DEFAULT_NEW_ALIGNMENT__); + SatC::deallocate(satc, p, n); + } + { + auto [p, n] = + SatC::allocate_at_least(satc, __STDCPP_DEFAULT_NEW_ALIGNMENT__ - 1); + VERIFY(n == __STDCPP_DEFAULT_NEW_ALIGNMENT__); + SatC::deallocate(satc, p, n); + } + { + auto [p, n] = SatC::allocate_at_least( + satc, __STDCPP_DEFAULT_NEW_ALIGNMENT__); + VERIFY(n == __STDCPP_DEFAULT_NEW_ALIGNMENT__); + SatC::deallocate(satc, p, n); + } + + using SatS = std::allocator_traits<std::allocator<short>>; + std::allocator<short> sats; + { + auto [p, n] = SatS::allocate_at_least(sats, 1); + VERIFY(n == __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(short)); + SatS::deallocate(sats, p, n); + } + { + auto [p, n] = SatS::allocate_at_least(sats, 2); + VERIFY(n == __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(short)); + SatS::deallocate(sats, p, n); + } + { + auto [p, n] = SatS::allocate_at_least(sats, + (__STDCPP_DEFAULT_NEW_ALIGNMENT__ - 1) / sizeof(short)); + VERIFY(n == __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(short)); + SatS::deallocate(sats, p, n); + } + { + auto [p, n] = SatS::allocate_at_least(sats, + __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(short)); + VERIFY(n == __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(short)); + SatS::deallocate(sats, p, n); + } + + struct A3 { char s[3]; }; + using SatA3 = std::allocator_traits<std::allocator<A3>>; + std::allocator<A3> sata3; + { + auto [p, n] = SatA3::allocate_at_least(sata3, 1); + VERIFY(n == __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(A3)); + SatA3::deallocate(sata3, p, n); + } + { + auto [p, n] = SatA3::allocate_at_least(sata3, 2); + VERIFY(n == __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(A3)); + SatA3::deallocate(sata3, p, n); + } + { + auto [p, n] = SatA3::allocate_at_least(sata3, + (__STDCPP_DEFAULT_NEW_ALIGNMENT__ - 1) / sizeof(A3)); + VERIFY(n == __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(A3)); + SatA3::deallocate(sata3, p, n); + } + { + auto [p, n] = SatA3::allocate_at_least(sata3, + __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(A3)); + VERIFY(n == __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(A3)); + SatA3::deallocate(sata3, p, n); + } + + struct Anm1 { char s[__STDCPP_DEFAULT_NEW_ALIGNMENT__ - 1]; }; + using SatAnm1 = std::allocator_traits<std::allocator<Anm1>>; + std::allocator<Anm1> satanm1; + { + auto [p, n] = SatAnm1::allocate_at_least(satanm1, 1); + VERIFY(n == 1); + SatAnm1::deallocate(satanm1, p, n); + } + { + auto [p, n] = SatAnm1::allocate_at_least(satanm1, + __STDCPP_DEFAULT_NEW_ALIGNMENT__); + VERIFY(n == __STDCPP_DEFAULT_NEW_ALIGNMENT__); + SatAnm1::deallocate(satanm1, p, n); + } + { + auto [p, n] = SatAnm1::allocate_at_least(satanm1, + __STDCPP_DEFAULT_NEW_ALIGNMENT__ - 1); + VERIFY(n == __STDCPP_DEFAULT_NEW_ALIGNMENT__); + SatAnm1::deallocate(satanm1, p, n); + } +} + +int main() +{ + base(); + extra(); +} diff --git a/libstdc++-v3/testsuite/21_strings/basic_string/capacity/char/18654.cc b/libstdc++-v3/testsuite/21_strings/basic_string/capacity/char/18654.cc index d542f34d08e6..59012a1c170b 100644 --- a/libstdc++-v3/testsuite/21_strings/basic_string/capacity/char/18654.cc +++ b/libstdc++-v3/testsuite/21_strings/basic_string/capacity/char/18654.cc @@ -58,7 +58,12 @@ void test01() #else str.shrink_to_fit(); // reserve is deprecated in C++20 #endif - VERIFY( str.capacity() == i ); +#if __glibcxx_allocate_at_least + unsigned limit = __STDCPP_DEFAULT_NEW_ALIGNMENT__ - 1; +#else + unsigned limit = 0; +#endif + VERIFY( str.capacity() - i <= limit); } } diff --git a/libstdc++-v3/testsuite/21_strings/basic_string/capacity/char/shrink_to_fit.cc b/libstdc++-v3/testsuite/21_strings/basic_string/capacity/char/shrink_to_fit.cc index a26d524dddf5..ae5df746f35d 100644 --- a/libstdc++-v3/testsuite/21_strings/basic_string/capacity/char/shrink_to_fit.cc +++ b/libstdc++-v3/testsuite/21_strings/basic_string/capacity/char/shrink_to_fit.cc @@ -30,7 +30,12 @@ void test01() s.push_back('b'); VERIFY( s.size() < s.capacity() ); s.shrink_to_fit(); - VERIFY( s.size() == s.capacity() ); +#ifdef __glibcxx_allocate_at_least + unsigned limit = __STDCPP_DEFAULT_NEW_ALIGNMENT__ - 1; +#else + unsigned limit = 0; +#endif + VERIFY( s.capacity() - s.size() <= limit ); } int main() diff --git a/libstdc++-v3/testsuite/21_strings/basic_string/capacity/wchar_t/18654.cc b/libstdc++-v3/testsuite/21_strings/basic_string/capacity/wchar_t/18654.cc index 49e45c764c48..bf3b8dd55814 100644 --- a/libstdc++-v3/testsuite/21_strings/basic_string/capacity/wchar_t/18654.cc +++ b/libstdc++-v3/testsuite/21_strings/basic_string/capacity/wchar_t/18654.cc @@ -50,6 +50,7 @@ void test01() const size_type cap = str.capacity(); VERIFY( cap >= 3 * i ); + // no shrink. str.reserve(2 * i); VERIFY( str.capacity() == cap ); @@ -58,7 +59,12 @@ void test01() #else str.shrink_to_fit(); // reserve is deprecated in C++20 #endif - VERIFY( str.capacity() == i ); +#if __glibcxx_allocate_at_least + unsigned limit = __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(wchar_t) - 1; +#else + unsigned limit = 0; +#endif + VERIFY( str.capacity() - i <= limit); } } diff --git a/libstdc++-v3/testsuite/21_strings/basic_string/capacity/wchar_t/2.cc b/libstdc++-v3/testsuite/21_strings/basic_string/capacity/wchar_t/2.cc index bff2bdd18629..04a3763fc778 100644 --- a/libstdc++-v3/testsuite/21_strings/basic_string/capacity/wchar_t/2.cc +++ b/libstdc++-v3/testsuite/21_strings/basic_string/capacity/wchar_t/2.cc @@ -27,14 +27,21 @@ void test02() { std::wstring str01 = L"twelve chars"; - // str01 becomes shared - std::wstring str02 = str01; + str01.reserve(100); #if __cplusplus <= 201703L str01.reserve(); #else str01.shrink_to_fit(); // reserve is deprecated in C++20 #endif - VERIFY( str01.capacity() == 12 ); + // These are not guaranteed to absolutely minimize storage. + // allocator<wchar_t>::allocate_at_least rounds up to what + // it knows ::op new delivers. +#ifdef __glibcxx_allocate_at_least + unsigned limit = __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(wchar_t) - 1; +#else + unsigned limit = 0; +#endif + VERIFY( str01.capacity() - str01.size() <= limit); } int main() diff --git a/libstdc++-v3/testsuite/21_strings/basic_string/capacity/wchar_t/shrink_to_fit.cc b/libstdc++-v3/testsuite/21_strings/basic_string/capacity/wchar_t/shrink_to_fit.cc index b4d0224a9d8f..fc0f659539f6 100644 --- a/libstdc++-v3/testsuite/21_strings/basic_string/capacity/wchar_t/shrink_to_fit.cc +++ b/libstdc++-v3/testsuite/21_strings/basic_string/capacity/wchar_t/shrink_to_fit.cc @@ -30,7 +30,12 @@ void test01() s.push_back(L'b'); VERIFY( s.size() < s.capacity() ); s.shrink_to_fit(); - VERIFY( s.size() == s.capacity() ); +#ifdef __glibcxx_allocate_at_least + unsigned limit = __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(wchar_t) - 1; +#else + unsigned limit = 0; +#endif + VERIFY( s.capacity() - s.size() <= limit ); } int main() diff --git a/libstdc++-v3/testsuite/23_containers/vector/capacity/shrink_to_fit.cc b/libstdc++-v3/testsuite/23_containers/vector/capacity/shrink_to_fit.cc index 74c68712b475..d4aa4d89b569 100644 --- a/libstdc++-v3/testsuite/23_containers/vector/capacity/shrink_to_fit.cc +++ b/libstdc++-v3/testsuite/23_containers/vector/capacity/shrink_to_fit.cc @@ -30,11 +30,12 @@ void test01() v.push_back(1); VERIFY( v.size() < v.capacity() ); v.shrink_to_fit(); -#if __cpp_exceptions - VERIFY( v.size() == v.capacity() ); +#ifdef __glibcxx_allocate_at_least + unsigned limit = __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(int); #else - VERIFY( v.size() < v.capacity() ); + unsigned limit = 0; #endif + VERIFY(v.capacity() - v.size() <= limit); } int main() diff --git a/libstdc++-v3/testsuite/23_containers/vector/capacity/shrink_to_fit2.cc b/libstdc++-v3/testsuite/23_containers/vector/capacity/shrink_to_fit2.cc index c8faa9ded803..9489e0438456 100644 --- a/libstdc++-v3/testsuite/23_containers/vector/capacity/shrink_to_fit2.cc +++ b/libstdc++-v3/testsuite/23_containers/vector/capacity/shrink_to_fit2.cc @@ -32,7 +32,12 @@ void test01() v.reserve(100); VERIFY( v.size() < v.capacity() ); v.shrink_to_fit(); - VERIFY( v.size() == v.capacity() ); +#ifdef __glibcxx_allocate_at_least + unsigned limit = __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(int); +#else + unsigned limit = 0; +#endif + VERIFY( v.capacity() - v.size() <= limit); VERIFY( v.get_allocator().get_personality() == alloc.get_personality() ); } @@ -45,7 +50,12 @@ void test02() v.reserve(100); VERIFY( v.size() < v.capacity() ); v.shrink_to_fit(); - VERIFY( v.size() == v.capacity() ); +#ifdef __glibcxx_allocate_at_least + unsigned limit = __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(int); +#else + unsigned limit = 0; +#endif + VERIFY( v.capacity() - v.size() <= limit); VERIFY( v.get_allocator().get_personality() == alloc.get_personality() ); } diff --git a/libstdc++-v3/testsuite/23_containers/vector/modifiers/emplace/self_emplace.cc b/libstdc++-v3/testsuite/23_containers/vector/modifiers/emplace/self_emplace.cc index 00a0c7b06ea8..629f35f05bac 100644 --- a/libstdc++-v3/testsuite/23_containers/vector/modifiers/emplace/self_emplace.cc +++ b/libstdc++-v3/testsuite/23_containers/vector/modifiers/emplace/self_emplace.cc @@ -99,37 +99,37 @@ struct A void test03() { - std::vector<A> va = - { - { A(1) }, - { A(2) }, - { A(3) } - }; - - // Make sure emplace will imply reallocation. - VERIFY( va.capacity() == 3 ); +#ifdef __glibcxx_allocate_at_least + unsigned fit = __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(A); +#else + unsigned fit = 4; +#endif + std::vector<A> va; va.reserve(fit); + for (int i = 1; va.size() < va.capacity(); ++i) + va.push_back(A(i)); va.emplace(va.begin(), va.begin()); - VERIFY( va.size() == 4 ); + VERIFY( va.size() == fit + 1 ); VERIFY( va[0]._i == 1 ); } void test04() { - std::vector<A> va = - { - { A(1) }, - { A(2) }, - { A(3) } - }; +#ifdef __glibcxx_allocate_at_least + unsigned fit = __STDCPP_DEFAULT_NEW_ALIGNMENT__ / sizeof(A); +#else + unsigned fit = 4; +#endif + std::vector<A> va; va.reserve(fit); + for (int i = 1; va.size() < va.capacity() - 1; ++i) + va.push_back(A(i)); // Make sure emplace won't reallocate. - va.reserve(4); va.emplace(va.begin(), va.begin()); - VERIFY( va.size() == 4 ); + VERIFY( va.size() == fit ); VERIFY( va[0]._i == 1 ); } diff --git a/libstdc++-v3/testsuite/util/testsuite_allocator.h b/libstdc++-v3/testsuite/util/testsuite_allocator.h index 892a385e307e..27372a97161d 100644 --- a/libstdc++-v3/testsuite/util/testsuite_allocator.h +++ b/libstdc++-v3/testsuite/util/testsuite_allocator.h @@ -168,6 +168,16 @@ namespace __gnu_test : Alloc(alloc) { } +#ifdef __glibcxx_allocate_at_least // C++23 + std::allocation_result<pointer, size_type> + allocate_at_least(size_type n) + { + auto [p, c] = AllocTraits::allocate_at_least(*this, n); + counter_type::allocate(c * sizeof(T)); + return { p, c }; + } +#endif + pointer allocate(size_type n, const void* = 0) { @@ -373,6 +383,32 @@ namespace __gnu_test return p; } +#ifdef __glibcxx_allocate_at_least + constexpr auto + allocate_at_least(size_type n) + -> std::allocation_result<Tp*, size_t> + { + auto r = AllocTraits::allocate_at_least(*this, n); + + if consteval + { return r; } + else + { + try + { + get_map().insert(map_type::value_type( + reinterpret_cast<void*>(r.ptr), personality)); + } + catch(...) + { + AllocTraits::deallocate(*this, r.ptr, r.count); + __throw_exception_again; + } + return r; + } + } +#endif + _GLIBCXX14_CONSTEXPR void deallocate(pointer p, size_type n) @@ -632,6 +668,13 @@ namespace __gnu_test pointer allocate(std::size_t n, const_void_pointer = {}) { return pointer(std::allocator<Tp>::allocate(n)); } +#ifdef __glibcxx_allocate_at_least + _GLIBCXX14_CONSTEXPR + std::allocation_result<pointer, std::size_t> + allocate_at_least(std::size_t n) + { return { allocate(n), n }; } +#endif + _GLIBCXX14_CONSTEXPR void deallocate(pointer p, std::size_t n) { std::allocator<Tp>::deallocate(std::addressof(*p), n); }
