================ @@ -0,0 +1,312 @@ +//===-- copyprof_allocator.cpp -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// This file implements the standalone memory allocator for CopyProf, managing +/// internal dynamic allocations required during profiling. +/// +//===----------------------------------------------------------------------===// + +#include "copyprof_allocator.h" + +#include <pthread.h> + +#include "copyprof_interceptors.h" +#include "copyprof_interface_internal.h" +#include "copyprof_shadow.h" +#include "sanitizer_common/sanitizer_allocator.h" +#include "sanitizer_common/sanitizer_allocator_checks.h" +#include "sanitizer_common/sanitizer_allocator_interface.h" +#include "sanitizer_common/sanitizer_allocator_report.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_errno.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_platform_limits_posix.h" +#include "sanitizer_common/sanitizer_stacktrace.h" + +namespace __copyprof { +namespace { + +using namespace __sanitizer; + +// Callbacks invoked by the sanitizer allocator whenever it maps or unmaps +// memory. They are no-ops, as CopyProf does not need to track the allocator's +// mappings. +struct CopyProfMapUnmapCallback { + // Called when the primary allocator maps memory for a region from which it + // serves small and medium sized allocations. + void OnMap(uptr p, uptr size) const {} + // Called when the secondary allocator maps memory for an allocation that is + // too large for the primary allocator. Each such allocation is mmap()ed + // separately. + void OnMapSecondary(uptr p, uptr size, uptr user_begin, + uptr user_size) const {} + // Called when either allocator unmaps memory. + void OnUnmap(uptr p, uptr size) const {} +}; + +struct CopyProfMetadata { + // The requested allocation size in bytes. + uptr requested_size; +}; + +struct AllocatorParameters { + static constexpr uptr kSpaceBeg = ~(uptr)0; + static constexpr uptr kSpaceSize = 0x40000000000ULL; + static constexpr uptr kMetadataSize = sizeof(CopyProfMetadata); + static constexpr uptr kFlags = 0; + using SizeClassMap = DefaultSizeClassMap; + using MapUnmapCallback = CopyProfMapUnmapCallback; + using AddressSpaceView = LocalAddressSpaceView; +}; + +using PrimaryAllocator = SizeClassAllocator64<AllocatorParameters>; +using Allocator = CombinedAllocator<PrimaryAllocator>; +using AllocatorCache = Allocator::AllocatorCache; + +// Per-thread allocator cache. It is mmap()ed, and thus zero-initialized as +// `AllocatorCache` requires, and committed back to `allocator` when its thread +// exits. As CopyProf does not intercept thread creation, the cache is created +// by the thread's first (de)allocation, which also arms the TSD destructor that +// commits it back. +pthread_key_t thread_cache_key; +__attribute__(( + tls_model("initial-exec"))) THREADLOCAL AllocatorCache* thread_cache; +// Set once the exiting thread's cache has been committed back. Later TSD +// destructors may still (de)allocate on this thread so they use the fallback +// cache, as a re-created cache could leak. +__attribute__(( + tls_model("initial-exec"))) THREADLOCAL bool thread_cache_committed_back; + +uptr ThreadCacheSize() { + return RoundUpTo(sizeof(AllocatorCache), GetPageSizeCached()); +} + +// Returns the calling thread's allocator cache, or `nullptr` if it has already +// been committed back. +AllocatorCache* GetThreadCache() { + if (LIKELY(thread_cache)) + return thread_cache; + if (UNLIKELY(thread_cache_committed_back)) + return nullptr; + thread_cache = + static_cast<AllocatorCache*>(MmapOrDie(ThreadCacheSize(), __func__)); + CHECK_EQ(0, pthread_setspecific(thread_cache_key, thread_cache)); + return thread_cache; +} + +// Custom allocator that keeps track of allocation sizes. +struct CopyProfAllocator { + // Heap allocations must be at least 8-byte aligned so independent + // allocations never share a shadow byte (avoiding data races). + static_assert(kMinAlignment == 1 << kShadowScale); + + Allocator allocator; + // Fallback allocator cache and mutex used in the slow path when no per-thread + // allocator cache is available. + StaticSpinMutex fallback_mutex; + AllocatorCache fallback_allocator_cache; + + void Init() { + SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); + allocator.Init(common_flags()->allocator_release_to_os_interval_ms); + } + + CopyProfMetadata* GetMetadata(const void* ptr) { + return reinterpret_cast<CopyProfMetadata*>(allocator.GetMetaData(ptr)); + } + + // `align` must be a non-zero power of two; callers are responsible for + // validating it. + void* Allocate(uptr size, uptr align = kMinAlignment) { + if (UNLIKELY(!copyprof_is_initialized)) + __copyprof_init_once(); + align = (align < kMinAlignment) ? kMinAlignment : align; + CHECK(IsPowerOfTwo(align)); + // C++ standard requires zero-sized allocations to return a non-null + // pointer. + if (size == 0) + size = 1; + void* allocated; + if (AllocatorCache* cache = GetThreadCache()) { + allocated = allocator.Allocate(cache, size, align); + } else { + SpinMutexLock lock(&fallback_mutex); + allocated = allocator.Allocate(&fallback_allocator_cache, size, align); + } + + if (UNLIKELY(!allocated)) { + SetAllocatorOutOfMemory(); + if (AllocatorMayReturnNull()) + return nullptr; + UNINITIALIZED BufferedStackTrace stack; + stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), + /*context=*/nullptr, common_flags()->fast_unwind_on_fatal); + ReportOutOfMemory(size, &stack); + } + GetMetadata(allocated)->requested_size = size; + return allocated; + } + + void Deallocate(void* ptr) { + if (UNLIKELY(!ptr)) + return; + if (AllocatorCache* cache = GetThreadCache()) { + allocator.Deallocate(cache, ptr); + } else { + SpinMutexLock lock(&fallback_mutex); + allocator.Deallocate(&fallback_allocator_cache, ptr); + } + } + + void* Calloc(uptr num, uptr size) { + if (UNLIKELY(CheckForCallocOverflow(size, num))) { + if (AllocatorMayReturnNull()) + return nullptr; + UNINITIALIZED BufferedStackTrace stack; + stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), + /*context=*/nullptr, common_flags()->fast_unwind_on_fatal); + ReportCallocOverflow(num, size, &stack); + } + void* ptr = Allocate(num * size); + // Memory from the secondary allocator is freshly mmap()ed, and thus already + // zeroed. + if (ptr && allocator.FromPrimary(ptr)) + REAL(memset)(ptr, 0, num * size); + return ptr; + } + + // Implemented via `Allocate` and `Deallocate` rather than + // `CombinedAllocator::Reallocate`, s.t. metadata is updated as for any other + // (de)allocation. + void* Reallocate(void* old_ptr, uptr new_size) { + CHECK(old_ptr && new_size); + void* new_ptr = Allocate(new_size); + if (new_ptr) { + CHECK_NE(REAL(memcpy), nullptr); + uptr memcpy_size = Min(new_size, GetMetadata(old_ptr)->requested_size); + REAL(memcpy)(new_ptr, old_ptr, memcpy_size); + Deallocate(old_ptr); + } + return new_ptr; + } +}; + +CopyProfAllocator instance; + +// TSD destructor that commits the exiting thread's cache back to `allocator`. +void CommitBackThreadCache(void* cache) { + thread_cache = nullptr; + thread_cache_committed_back = true; + instance.allocator.SwallowCache(static_cast<AllocatorCache*>(cache)); + UnmapOrDie(cache, ThreadCacheSize()); +} + +} // namespace + +void InitializeCopyProfAllocator() { + instance.Init(); + CHECK_EQ(0, pthread_key_create(&thread_cache_key, CommitBackThreadCache)); +} + +void* CopyProfAllocate(uptr size, uptr align) { + if (UNLIKELY(align == 0 || !IsPowerOfTwo(align))) { + errno = errno_EINVAL; + if (AllocatorMayReturnNull()) + return nullptr; + UNINITIALIZED BufferedStackTrace stack; + stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), + /*context=*/nullptr, common_flags()->fast_unwind_on_fatal); + ReportInvalidAllocationAlignment(align, &stack); + } + return SetErrnoOnNull(instance.Allocate(size, align)); +} + +void CopyProfDeallocate(void* ptr, uptr size, uptr align) { ---------------- snehasish wrote:
Maybe drop the names of unused params? https://github.com/llvm/llvm-project/pull/235453 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
