Drivers walking a manager's LRU with ttm_bo_lru_for_each_reserved_guarded() may decide that the rest of the current priority isn't worth walking, for example when the priority is roughly age ordered and the walk hit a BO that is still too hot to evict. Today the only option is to stop the walk altogether, which also gives up on the remaining priorities.
Add ttm_resource_manager_next_priority() to move a resource cursor to the start of the next priority, and ttm_bo_lru_cursor_skip_priority() to make the next ttm_bo_lru_cursor_next() use it. While at it, stop incrementing the cursor priority past the last valid priority once the iteration is done. Assisted-by: LLM Signed-off-by: Matthew Brost <[email protected]> --- drivers/gpu/drm/ttm/ttm_bo_util.c | 3 +++ drivers/gpu/drm/ttm/ttm_resource.c | 35 ++++++++++++++++++++++++++---- include/drm/ttm/ttm_bo.h | 19 ++++++++++++++++ include/drm/ttm/ttm_resource.h | 2 ++ 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index f9458a4d5a09..d626f014e61f 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -997,6 +997,9 @@ __ttm_bo_lru_cursor_next(struct ttm_bo_lru_cursor *curs) if (first) { res = ttm_resource_manager_first(&curs->res_curs); first = false; + } else if (curs->skip_priority) { + res = ttm_resource_manager_next_priority(&curs->res_curs); + curs->skip_priority = false; } else { res = ttm_resource_manager_next(&curs->res_curs); } diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c index 4e6d3d658d85..e631507d7aa1 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -674,6 +674,17 @@ ttm_resource_cursor_check_bulk(struct ttm_resource_cursor *cursor, } } +static bool ttm_resource_cursor_next_priority(struct ttm_resource_cursor *cursor) +{ + if (cursor->priority + 1 >= TTM_MAX_BO_PRIORITY) + return false; + + list_move(&cursor->hitch.link, &cursor->man->lru[++cursor->priority]); + ttm_resource_cursor_clear_bulk(cursor); + + return true; +} + /** * ttm_resource_manager_first() - Start iterating over the resources * of a resource manager @@ -723,16 +734,32 @@ ttm_resource_manager_next(struct ttm_resource_cursor *cursor) } } - if (++cursor->priority >= TTM_MAX_BO_PRIORITY) + if (!ttm_resource_cursor_next_priority(cursor)) break; - - list_move(&cursor->hitch.link, &man->lru[cursor->priority]); - ttm_resource_cursor_clear_bulk(cursor); } return NULL; } +/** + * ttm_resource_manager_next_priority() - Skip the rest of the current + * priority and continue iterating at the next one + * @cursor: cursor to record the position + * + * Return: the first resource of the next non-empty priority, or NULL if + * there is none. + */ +struct ttm_resource * +ttm_resource_manager_next_priority(struct ttm_resource_cursor *cursor) +{ + lockdep_assert_held(&cursor->man->bdev->lru_lock); + + if (!ttm_resource_cursor_next_priority(cursor)) + return NULL; + + return ttm_resource_manager_next(cursor); +} + /** * ttm_lru_first_res_or_null() - Return the first resource on an lru list * @head: The list head of the lru list. diff --git a/include/drm/ttm/ttm_bo.h b/include/drm/ttm/ttm_bo.h index 32791c4db2a9..2fc517dbf03d 100644 --- a/include/drm/ttm/ttm_bo.h +++ b/include/drm/ttm/ttm_bo.h @@ -507,10 +507,29 @@ struct ttm_bo_lru_cursor { * unlock before the next iteration or after loop exit. */ bool needs_unlock; + /** + * @skip_priority: Continue at the next LRU priority on the next + * iteration. See ttm_bo_lru_cursor_skip_priority(). + */ + bool skip_priority; /** @arg: Pointer to common BO LRU walk arguments. */ struct ttm_lru_walk_arg *arg; }; +/** + * ttm_bo_lru_cursor_skip_priority() - Skip the rest of the current priority + * @curs: The cursor. + * + * Make the next ttm_bo_lru_cursor_next() skip the remaining buffer objects + * of the current LRU priority and continue at the next priority, for + * example when the caller decides the rest of the priority isn't worth + * walking. + */ +static inline void ttm_bo_lru_cursor_skip_priority(struct ttm_bo_lru_cursor *curs) +{ + curs->skip_priority = true; +} + void ttm_bo_lru_cursor_fini(struct ttm_bo_lru_cursor *curs); struct ttm_bo_lru_cursor * diff --git a/include/drm/ttm/ttm_resource.h b/include/drm/ttm/ttm_resource.h index 714c533622c2..8cd70d1e5481 100644 --- a/include/drm/ttm/ttm_resource.h +++ b/include/drm/ttm/ttm_resource.h @@ -499,6 +499,8 @@ struct ttm_resource * ttm_resource_manager_first(struct ttm_resource_cursor *cursor); struct ttm_resource * ttm_resource_manager_next(struct ttm_resource_cursor *cursor); +struct ttm_resource * +ttm_resource_manager_next_priority(struct ttm_resource_cursor *cursor); struct ttm_resource * ttm_lru_first_res_or_null(struct list_head *head); -- 2.34.1
