BOs whose backing TT pages were allocated below the device's beneficial order (tracked via the TTM_TT_FLAG_BENEFICIAL_ORDER_FAILED tt page flag) are added to a per-device defrag list. Add a delayed worker that periodically walks this list and attempts to re-back each BO with beneficial-order pages, improving TLB efficiency for long-lived objects that were initially allocated under memory pressure.
The worker is kicked when the list first transitions to non-empty and reschedules itself while work remains. On a failed pass it backs off exponentially (XE_BO_DEFRAG_INTERVAL_MS up to XE_BO_DEFRAG_INTERVAL_MAX_MS) and retries the whole list later; the interval is reset to the default on the next fresh enqueue. The worker stops scheduling once the list drains. A defrag move synchronously reallocates and re-copies a BO's backing store, which is not free. To avoid stalling concurrent active work when many BOs (or one very large BO) become eligible at once (e.g. after a burst of memory pressure, which would otherwise manifest as a large FPS drop while the worker churns through the whole list), cap the number of newly (re)allocated bytes processed per run to XE_BO_DEFRAG_SIZE_LIMIT and reschedule. Only the pages a move actually reallocates are charged - pages harvested (borrowed) from the old backing are free and do not count - so the budget tracks real defrag work and an object larger than the budget is upgraded in budget-sized slices across runs. This spreads the defrag effort over time, keeping it in the background and yielding to userspace progress. Defragmenting a BO forces a TTM move (ctx.defrag) that reallocates the backing at the beneficial order and relocates the contents on the GPU (see xe_bo_move()). The per-BO move is bounded by the run's remaining byte budget via ctx.defrag_bytes_remaining; the pool allocator reports how much it spent so the worker can charge it accurately. When beneficial-order pages cannot be obtained the new allocation is discharged during populate and the BO keeps its original backing, staying on the list for a later retry. Cc: Carlos Santa <[email protected]> Cc: Ryan Neph <[email protected]> Cc: Christian Koenig <[email protected]> Cc: Huang Rui <[email protected]> Cc: Matthew Auld <[email protected]> Cc: Maarten Lankhorst <[email protected]> Cc: Maxime Ripard <[email protected]> Cc: Thomas Zimmermann <[email protected]> Cc: David Airlie <[email protected]> Cc: Simona Vetter <[email protected]> Cc: [email protected] Cc: [email protected] Cc: Thomas Hellström <[email protected]> Assisted-by: GitHub_Copilot:claude-opus-4.8 Signed-off-by: Matthew Brost <[email protected]> --- drivers/gpu/drm/xe/xe_bo.c | 277 ++++++++++++++++++++++++++- drivers/gpu/drm/xe/xe_bo.h | 1 + drivers/gpu/drm/xe/xe_device.c | 9 + drivers/gpu/drm/xe/xe_device_types.h | 11 ++ 4 files changed, 294 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 3b867c486ea4..6a847e42946c 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -31,6 +31,7 @@ #include "xe_pat.h" #include "xe_pm.h" #include "xe_preempt_fence.h" +#include "xe_printk.h" #include "xe_pxp.h" #include "xe_res_cursor.h" #include "xe_shrinker.h" @@ -49,6 +50,35 @@ */ #define XE_BO_DEFRAG_RECLAIM_BACKOFF_THRESHOLD 2 +/* + * Maximum number of bytes of newly (re)allocated backing the defrag worker will + * produce in a single run before yielding and rescheduling itself. + * + * A defrag move synchronously reallocates and re-copies a BO's backing store, + * which is not free. If a large number of BOs (or one very large BO) become + * eligible at once (e.g. after a burst of memory pressure), processing them all + * in one worker run would hold things up for a long, unbounded stretch and can + * visibly starve concurrent active work (e.g. a large FPS drop). Instead, cap + * the bytes actually upgraded per run and requeue, spreading the defrag effort + * out over time so it stays in the background and yields to userspace progress. + * Only pages a move truly reallocates are charged; pages harvested from the old + * backing are free, so an object larger than the budget is upgraded in + * budget-sized slices across runs. + */ +#define XE_BO_DEFRAG_SIZE_LIMIT SZ_32M + +/* Default delay before (re)running the defrag worker, in milliseconds. */ +#define XE_BO_DEFRAG_INTERVAL_MS 25 + +/* + * Upper bound for the (exponentially backed off) defrag worker interval, in + * milliseconds, so repeated failures don't push the retry arbitrarily far out. + */ +#define XE_BO_DEFRAG_INTERVAL_MAX_MS 15000 /* 15 seconds */ + +static void xe_bo_defrag_worker(struct work_struct *w); +static void xe_place_from_ttm_type(u32 mem_type, struct ttm_place *place); + const char *const xe_mem_type_to_name[TTM_NUM_MEM_TYPES] = { [XE_PL_SYSTEM] = "system", [XE_PL_TT] = "gtt", @@ -624,8 +654,10 @@ static int xe_ttm_tt_populate(struct ttm_device *ttm_dev, struct ttm_tt *tt, } else { struct xe_device *xe = ttm_to_xe_device(ttm_dev); - if (atomic_read(&xe->mem.defrag.count) >= - XE_BO_DEFRAG_RECLAIM_BACKOFF_THRESHOLD) + if (ctx->defrag) + ctx->beneficial_reclaim_backoff = false; + else if (atomic_read(&xe->mem.defrag.count) >= + XE_BO_DEFRAG_RECLAIM_BACKOFF_THRESHOLD) ctx->beneficial_reclaim_backoff = true; ttm_tt_clear_backed_up(tt); @@ -1041,18 +1073,54 @@ static int xe_ttm_bo_purge(struct ttm_buffer_object *ttm_bo, struct ttm_operatio return 0; } +static void xe_bo_defrag_fini(void *arg) +{ + struct xe_device *xe = arg; + + disable_delayed_work_sync(&xe->mem.defrag.worker); +} + /** * xe_bo_defrag_init_early() - Initialize the device defrag BO tracking * @xe: The xe device * - * Initialize the list, lock and count used to track BOs whose backing TT - * pages were allocated at a sub-optimal order. + * Initialize the list, lock, count and delayed worker used to track and + * defragment BOs whose backing TT pages were allocated at a sub-optimal order. + * + * The devm action that cancels the worker on teardown is registered separately + * by xe_bo_defrag_init(), which must run after the migrate contexts the + * worker depends on have been initialized. */ void xe_bo_defrag_init_early(struct xe_device *xe) { spin_lock_init(&xe->mem.defrag.lock); INIT_LIST_HEAD(&xe->mem.defrag.list); atomic_set(&xe->mem.defrag.count, 0); + xe->mem.defrag.interval_ms = XE_BO_DEFRAG_INTERVAL_MS; + INIT_DELAYED_WORK(&xe->mem.defrag.worker, xe_bo_defrag_worker); +} + +/** + * xe_bo_defrag_init() - Register defrag worker teardown + * @xe: The xe device + * + * Register the devm action that disables and drains the defrag worker on + * device teardown. This must be called after xe_migrate_init() so that, with + * devm's reverse-order cleanup, the worker is stopped before the migrate + * contexts it relies on (via xe_bo_defrag_one() -> ttm_bo_validate() -> + * xe_bo_move() -> xe_migrate_copy_defrag()) are torn down. + * + * Return: 0 on success, negative error code on failure. + */ +int xe_bo_defrag_init(struct xe_device *xe) +{ + return devm_add_action_or_reset(xe->drm.dev, xe_bo_defrag_fini, xe); +} + +static void xe_bo_defrag_schedule(struct xe_device *xe) +{ + schedule_delayed_work(&xe->mem.defrag.worker, + msecs_to_jiffies(xe->mem.defrag.interval_ms)); } static bool xe_bo_needs_defrag(struct xe_bo *bo) @@ -1073,16 +1141,24 @@ static bool xe_bo_needs_defrag(struct xe_bo *bo) static void xe_bo_defrag_add(struct xe_bo *bo) { struct xe_device *xe = xe_bo_device(bo); + bool kick = false; xe_bo_assert_held(bo); xe_assert(xe, xe_bo_needs_defrag(bo)); scoped_guard(spinlock, &xe->mem.defrag.lock) { if (list_empty(&bo->defrag_link)) { + /* Kick the worker when the list transitions to non-empty. */ + kick = list_empty(&xe->mem.defrag.list); + if (kick) + xe->mem.defrag.interval_ms = XE_BO_DEFRAG_INTERVAL_MS; list_add_tail(&bo->defrag_link, &xe->mem.defrag.list); atomic_inc(&xe->mem.defrag.count); } } + + if (kick) + xe_bo_defrag_schedule(xe); } static void __xe_bo_defrag_remove(struct xe_bo *bo) @@ -1125,6 +1201,199 @@ static void xe_bo_defrag_update(struct xe_bo *bo) xe_bo_defrag_remove(bo); } +/* + * Attempt to defragment a single BO by forcing a move that reallocates its + * backing at the device's beneficial order. @budget bounds how many bytes of + * new pages the move may (re)allocate; the remainder of the object is completed + * by harvesting the old backing, and the BO stays flagged for a later pass. On + * return *@consumed holds the bytes actually (re)allocated (0 on failure or + * when nothing needed reallocating). Returns 0 if the BO no longer needs to be + * tracked (either defragmented or no longer eligible), or a negative error code + * if the attempt should be retried later. + */ +static int xe_bo_defrag_one(struct xe_device *xe, struct xe_bo *bo, + u64 budget, u64 *consumed) +{ + struct ttm_operation_ctx ctx = { + .gfp_retry_mayfail = true, + .defrag = true, + .defrag_bytes_remaining = budget, + }; + struct ttm_buffer_object *ttm_bo = &bo->ttm; + struct ttm_placement placement; + struct ttm_place place; + int ret = 0; + + *consumed = 0; + + xe_bo_lock(bo, false); + + /* Re-check eligibility under the BO lock. */ + if (!xe_bo_needs_defrag(bo)) { + xe_bo_defrag_remove(bo); + goto unlock; + } + + xe_place_from_ttm_type(ttm_bo->resource->mem_type, &place); + place.flags |= TTM_PL_FLAG_TEMPORARY; + placement.num_placement = 1; + placement.placement = &place; + + /* + * On success the move reallocates the backing at beneficial order and + * drops the BO from the defrag list. On failure the BO keeps its + * original backing and stays on the list for a later retry. + */ + ret = ttm_bo_validate(ttm_bo, &placement, &ctx); + + /* + * The pool allocator decremented defrag_bytes_remaining as it + * (re)allocated pages, so the unused budget tells us how much was spent. + * Only meaningful when the move succeeded. A successful move may spend + * nothing - e.g. when the object was already fully upgraded and every + * chunk is harvested (borrowed) from the old backing for free - so a + * zero spend is legitimate and must be charged as zero, not the whole + * budget. + */ + if (!ret) { + s64 spent = (s64)budget - ctx.defrag_bytes_remaining; + + *consumed = spent > 0 ? spent : 0; + } + + xe_dbg(xe, "Defrag attempt on BO size=%zu: ret=%pe consumed=%llu\n", + xe_bo_size(bo), ERR_PTR(ret), *consumed); + +unlock: + xe_bo_unlock(bo); + return ret; +} + +static void xe_bo_defrag_worker(struct work_struct *w) +{ + struct delayed_work *dwork = to_delayed_work(w); + struct xe_device *xe = + container_of(dwork, struct xe_device, mem.defrag.worker); + u64 defrag_bytes = 0; + bool requeue = false; + int idx; + + if (!drm_dev_enter(&xe->drm, &idx)) + return; + + if (!xe_pm_runtime_get_if_in_use(xe)) { + drm_dev_exit(idx); + return; + } + + /* + * Process at most XE_BO_DEFRAG_SIZE_LIMIT bytes of newly (re)allocated + * backing per run rather than draining the whole list in one go. Only + * the pages a move actually reallocates are charged - pages harvested + * (borrowed) from the old backing are free and do not count - so the + * budget tracks real defrag work. A single object larger than the + * budget is upgraded in slices across successive runs. Each defrag is + * synchronous and relatively expensive, so bounding the work per run + * keeps the worker from monopolising resources and lets concurrent + * active work make progress; any remaining work is handled by a + * follow-up run scheduled below. + */ + while (defrag_bytes < XE_BO_DEFRAG_SIZE_LIMIT) { + struct xe_bo *first, *bo; + u64 consumed = 0; + int ret; + + scoped_guard(spinlock, &xe->mem.defrag.lock) { + bool again; + + /* Progress, reset interval */ + if (defrag_bytes) + xe->mem.defrag.interval_ms = XE_BO_DEFRAG_INTERVAL_MS; + + do { + first = list_first_entry_or_null(&xe->mem.defrag.list, + struct xe_bo, + defrag_link); + bo = first ? xe_bo_get_unless_zero(first) : NULL; + again = first && !bo; + if (again) { + list_del_init(&first->defrag_link); + atomic_dec(&xe->mem.defrag.count); + } + } while (again); + } + + if (!bo) + break; + + /* + * Bound the per-BO move to the run's remaining byte budget. A + * large object is upgraded in budget-sized slices across runs; + * the slices already upgraded come back as free already- + * beneficial harvests, so each run only spends budget on new + * forward progress. + */ + ret = xe_bo_defrag_one(xe, bo, XE_BO_DEFRAG_SIZE_LIMIT - defrag_bytes, + &consumed); + defrag_bytes += consumed; + + if (ret || ttm_tt_is_beneficial_order_failed(bo->ttm.ttm)) { + scoped_guard(spinlock, &xe->mem.defrag.lock) { + if (ret) + /* + * Abort the pass and retry the whole + * list later, backing off exponentially + * on every failure. + */ + xe->mem.defrag.interval_ms = + min(xe->mem.defrag.interval_ms * 2, + (unsigned int)XE_BO_DEFRAG_INTERVAL_MAX_MS); + else + /* Progress, reset interval */ + xe->mem.defrag.interval_ms = XE_BO_DEFRAG_INTERVAL_MS; + } + + xe_bo_put(bo); + requeue = true; + break; + } + + xe_bo_put(bo); + } + + /* + * Decide whether to reschedule: + * + * - The loop hit its per-run limit (defrag_bytes >= + * XE_BO_DEFRAG_SIZE_LIMIT): we stopped early to spread the work out, + * so requeue at the default interval (reset here since this run made + * progress) whenever the list still has entries. + * + * - The loop aborted on a failed defrag (requeue): retry later at the + * backed-off interval computed above. + * + * - The loop drained the list (broke on the empty list): nothing left to + * do, so stop scheduling. A fresh enqueue will kick the worker again. + */ + if (defrag_bytes >= XE_BO_DEFRAG_SIZE_LIMIT) { + struct xe_bo *bo; + + scoped_guard(spinlock, &xe->mem.defrag.lock) { + bo = list_first_entry_or_null(&xe->mem.defrag.list, + struct xe_bo, defrag_link); + xe->mem.defrag.interval_ms = XE_BO_DEFRAG_INTERVAL_MS; + } + + if (bo) + xe_bo_defrag_schedule(xe); + } else if (requeue) { + xe_bo_defrag_schedule(xe); + } + + xe_pm_runtime_put(xe); + drm_dev_exit(idx); +} + static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, struct ttm_operation_ctx *ctx, struct ttm_resource *new_mem, diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index 32211016a722..38d50f8122cb 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -226,6 +226,7 @@ int xe_bo_pin(struct xe_bo *bo, struct drm_exec *exec); void xe_bo_unpin_external(struct xe_bo *bo); void xe_bo_unpin(struct xe_bo *bo); void xe_bo_defrag_init_early(struct xe_device *xe); +int xe_bo_defrag_init(struct xe_device *xe); int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict, struct drm_exec *exec); diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index d4321419ee35..df5bb94f3ecb 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -1063,6 +1063,15 @@ int xe_device_probe(struct xe_device *xe) return err; } + /* + * Register the defrag worker teardown now that the migrate contexts it + * depends on are initialized, so devm's reverse-order cleanup stops the + * worker before those contexts are torn down. + */ + err = xe_bo_defrag_init(xe); + if (err) + return err; + err = xe_pagefault_init(xe); if (err) return err; diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 9e6244d8f41b..6e68add4d652 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -314,6 +314,17 @@ struct xe_device { * @mem.defrag.list. */ atomic_t count; + /** + * @mem.defrag.worker: Delayed worker that walks + * @mem.defrag.list trying to reallocate BO backing + * store at the device's beneficial order. + */ + struct delayed_work worker; + /** + * @mem.defrag.interval_ms: Reschedule interval for + * @mem.defrag.worker, in milliseconds. + */ + unsigned int interval_ms; } defrag; } mem; -- 2.34.1
