The branch main has been updated by markj: URL: https://cgit.FreeBSD.org/src/commit/?id=6337ca19a3637aa72eddf4d62a2eaf7d8df51638
commit 6337ca19a3637aa72eddf4d62a2eaf7d8df51638 Author: Mark Johnston <[email protected]> AuthorDate: 2026-07-24 21:11:58 +0000 Commit: Mark Johnston <[email protected]> CommitDate: 2026-07-24 21:11:58 +0000 uma: Factor out the implementations of uma_zfree_{arg,smr}() The two function both free an item to a UMA zone, but uma_zfree_arg() does so in such as way as to ensure that the item will be the first one returned by a subsequent allocation, while uma_zfree_smr() must defer reuse of the item and therefore never frees to the per-CPU alloc bucket. When KASAN is enabled, we actually want uma_zfree_arg() to behave like uma_zfree_smr(): to improve the reliability of use-after-free detection, reuse of the newly freed item should be deferred for some time. Refactor a bit to make it easier to improve KASAN along these lines: introduce two helper functions, cache_free_item() and cache_free_smr(), which handle most of the work of interacting with the per-CPU caches. A subsequent commit will let uma_zfree_arg() use cache_free_smr() when KASAN is enabled. No functional change intended. Reviewed by: rlibby MFC after: 1 month Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D58268 --- sys/vm/uma_core.c | 159 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 91 insertions(+), 68 deletions(-) diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c index 410b4f5a5142..02b2df89f636 100644 --- a/sys/vm/uma_core.c +++ b/sys/vm/uma_core.c @@ -4447,37 +4447,82 @@ fail: return (NULL); } -/* See uma.h */ -void -uma_zfree_smr(uma_zone_t zone, void *item) +static __always_inline bool +cache_free_item(uma_zone_t zone, int uz_flags, void *item, void *udata) { uma_cache_t cache; - uma_cache_bucket_t bucket; int itemdomain; + + /* + * If possible, free to the per-CPU cache. There are two + * requirements for safe access to the per-CPU cache: (1) the thread + * accessing the cache must not be preempted or yield during access, + * and (2) the thread must not migrate CPUs without switching which + * cache it accesses. We rely on a critical section to prevent + * preemption and migration. We release the critical section in + * order to acquire the zone mutex if we are unable to free to the + * current cache; when we re-acquire the critical section, we must + * detect and handle migration if it has occurred. + */ + itemdomain = 0; #ifdef NUMA - int uz_flags; + if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) + itemdomain = item_domain(item); #endif - CTR3(KTR_UMA, "uma_zfree_smr zone %s(%p) item %p", - zone->uz_name, zone, item); + critical_enter(); + do { + uma_cache_bucket_t bucket; -#ifdef UMA_ZALLOC_DEBUG - KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, - ("uma_zfree_smr: called with non-SMR zone.")); - KASSERT(item != NULL, ("uma_zfree_smr: Called with NULL pointer.")); - SMR_ASSERT_NOT_ENTERED(zone->uz_smr); - if (uma_zfree_debug(zone, item, NULL) == EJUSTRETURN) - return; + cache = &zone->uz_cpu[curcpu]; + /* + * Try to free into the allocbucket first to give LIFO + * ordering for cache-hot datastructures. Spill over + * into the freebucket if necessary. Alloc will swap + * them if one runs dry. + */ + bucket = &cache->uc_allocbucket; +#ifdef NUMA + if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && + PCPU_GET(domain) != itemdomain) { + bucket = &cache->uc_crossbucket; + } else #endif - cache = &zone->uz_cpu[curcpu]; + if (bucket->ucb_cnt == bucket->ucb_entries && + cache->uc_freebucket.ucb_cnt < + cache->uc_freebucket.ucb_entries) + cache_bucket_swap(&cache->uc_freebucket, + &cache->uc_allocbucket); + if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) { + cache_bucket_push(cache, bucket, item); + critical_exit(); + return (true); + } + } while (cache_free(zone, cache, udata, itemdomain)); + critical_exit(); + + return (false); +} + +static __always_inline bool +cache_free_smr(uma_zone_t zone, void *item, void *udata) +{ + uma_cache_t cache; + int itemdomain; +#ifdef NUMA + int uz_flags; +#endif + itemdomain = 0; #ifdef NUMA - uz_flags = cache_uz_flags(cache); + uz_flags = cache_uz_flags(&zone->uz_cpu[curcpu]); if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) itemdomain = item_domain(item); #endif critical_enter(); do { + uma_cache_bucket_t bucket; + cache = &zone->uz_cpu[curcpu]; /* SMR Zones must free to the free bucket. */ bucket = &cache->uc_freebucket; @@ -4490,11 +4535,33 @@ uma_zfree_smr(uma_zone_t zone, void *item) if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) { cache_bucket_push(cache, bucket, item); critical_exit(); - return; + return (true); } - } while (cache_free(zone, cache, NULL, itemdomain)); + } while (cache_free(zone, cache, udata, itemdomain)); critical_exit(); + return (false); +} + +/* See uma.h */ +void +uma_zfree_smr(uma_zone_t zone, void *item) +{ + CTR3(KTR_UMA, "uma_zfree_smr zone %s(%p) item %p", + zone->uz_name, zone, item); + +#ifdef UMA_ZALLOC_DEBUG + KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, + ("uma_zfree_smr: called with non-SMR zone.")); + KASSERT(item != NULL, ("uma_zfree_smr: Called with NULL pointer.")); + SMR_ASSERT_NOT_ENTERED(zone->uz_smr); + if (uma_zfree_debug(zone, item, NULL) == EJUSTRETURN) + return; +#endif + + if (cache_free_smr(zone, item, NULL)) + return; + /* * If nothing else caught this, we'll just do an internal free. */ @@ -4506,8 +4573,7 @@ void uma_zfree_arg(uma_zone_t zone, void *item, void *udata) { uma_cache_t cache; - uma_cache_bucket_t bucket; - int itemdomain, uz_flags; + int uz_flags; /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); @@ -4540,55 +4606,12 @@ uma_zfree_arg(uma_zone_t zone, void *item, void *udata) * The race here is acceptable. If we miss it we'll just have to wait * a little longer for the limits to be reset. */ - if (__predict_false(uz_flags & UMA_ZFLAG_LIMIT)) { - if (atomic_load_32(&zone->uz_sleepers) > 0) - goto zfree_item; - } + if (__predict_false(uz_flags & UMA_ZFLAG_LIMIT) && + atomic_load_32(&zone->uz_sleepers) > 0) + goto zfree_item; - /* - * If possible, free to the per-CPU cache. There are two - * requirements for safe access to the per-CPU cache: (1) the thread - * accessing the cache must not be preempted or yield during access, - * and (2) the thread must not migrate CPUs without switching which - * cache it accesses. We rely on a critical section to prevent - * preemption and migration. We release the critical section in - * order to acquire the zone mutex if we are unable to free to the - * current cache; when we re-acquire the critical section, we must - * detect and handle migration if it has occurred. - */ - itemdomain = 0; -#ifdef NUMA - if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) - itemdomain = item_domain(item); -#endif - critical_enter(); - do { - cache = &zone->uz_cpu[curcpu]; - /* - * Try to free into the allocbucket first to give LIFO - * ordering for cache-hot datastructures. Spill over - * into the freebucket if necessary. Alloc will swap - * them if one runs dry. - */ - bucket = &cache->uc_allocbucket; -#ifdef NUMA - if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && - PCPU_GET(domain) != itemdomain) { - bucket = &cache->uc_crossbucket; - } else -#endif - if (bucket->ucb_cnt == bucket->ucb_entries && - cache->uc_freebucket.ucb_cnt < - cache->uc_freebucket.ucb_entries) - cache_bucket_swap(&cache->uc_freebucket, - &cache->uc_allocbucket); - if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) { - cache_bucket_push(cache, bucket, item); - critical_exit(); - return; - } - } while (cache_free(zone, cache, udata, itemdomain)); - critical_exit(); + if (cache_free_item(zone, uz_flags, item, udata)) + return; /* * If nothing else caught this, we'll just do an internal free.
