Drivers walking a manager's LRU with ttm_bo_lru_for_each_reserved_guarded() may decide that the rest of the current priority isn't worth walking, for example when the priority is roughly age ordered and the walk hit a BO that is still too hot to evict. Today the only option is to stop the walk altogether, which also gives up on the remaining priorities.
Add ttm_resource_cursor_skip_priority(), which parks the cursor hitch at the end of the current priority. The iteration already copes with that: it finds nothing left in the priority and moves on to the next one, or ends if there is none. Wrap it in ttm_bo_lru_cursor_skip_priority() for users of the bo LRU cursor. While at it, stop incrementing the cursor priority past the last valid priority once the iteration is done, as it is now used to index the lru array outside of the iteration itself. Assisted-by: LLM Signed-off-by: Matthew Brost <[email protected]> Reviewed-by: Thomas Hellström <[email protected]> --- v7: - Drop extra bool rather just advance hitch (Natalie) --- drivers/gpu/drm/ttm/ttm_bo_util.c | 19 +++++++++++++++++ drivers/gpu/drm/ttm/ttm_resource.c | 33 ++++++++++++++++++++++++++---- include/drm/ttm/ttm_bo.h | 2 ++ include/drm/ttm/ttm_resource.h | 1 + 4 files changed, 51 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index f9458a4d5a09..d81dd3738dc0 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -955,6 +955,25 @@ void ttm_bo_lru_cursor_fini(struct ttm_bo_lru_cursor *curs) } EXPORT_SYMBOL(ttm_bo_lru_cursor_fini); +/** + * ttm_bo_lru_cursor_skip_priority() - Skip the rest of the current priority + * @curs: The cursor. + * + * Make the next ttm_bo_lru_cursor_next() skip the remaining buffer objects + * of the current LRU priority and continue at the next one, for example + * when the caller decides the rest of the priority isn't worth walking. + * If the current priority is the last one, the iteration ends. + */ +void ttm_bo_lru_cursor_skip_priority(struct ttm_bo_lru_cursor *curs) +{ + struct ttm_device *bdev = curs->res_curs.man->bdev; + + spin_lock(&bdev->lru_lock); + ttm_resource_cursor_skip_priority(&curs->res_curs); + spin_unlock(&bdev->lru_lock); +} +EXPORT_SYMBOL(ttm_bo_lru_cursor_skip_priority); + /** * ttm_bo_lru_cursor_init() - Initialize a struct ttm_bo_lru_cursor * @curs: The ttm_bo_lru_cursor to initialize. diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c index 4e6d3d658d85..2db5d9bbfbe7 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -674,6 +674,17 @@ ttm_resource_cursor_check_bulk(struct ttm_resource_cursor *cursor, } } +static bool ttm_resource_cursor_next_priority(struct ttm_resource_cursor *cursor) +{ + if (cursor->priority + 1 >= TTM_MAX_BO_PRIORITY) + return false; + + list_move(&cursor->hitch.link, &cursor->man->lru[++cursor->priority]); + ttm_resource_cursor_clear_bulk(cursor); + + return true; +} + /** * ttm_resource_manager_first() - Start iterating over the resources * of a resource manager @@ -723,16 +734,30 @@ ttm_resource_manager_next(struct ttm_resource_cursor *cursor) } } - if (++cursor->priority >= TTM_MAX_BO_PRIORITY) + if (!ttm_resource_cursor_next_priority(cursor)) break; - - list_move(&cursor->hitch.link, &man->lru[cursor->priority]); - ttm_resource_cursor_clear_bulk(cursor); } return NULL; } +/** + * ttm_resource_cursor_skip_priority() - Skip the rest of the current priority + * @cursor: cursor to record the position + * + * Make the next ttm_resource_manager_next() continue at the next priority, + * or end the iteration if the current priority is the last one. Done by + * parking the hitch at the end of the current priority, which the iteration + * already handles. + */ +void ttm_resource_cursor_skip_priority(struct ttm_resource_cursor *cursor) +{ + lockdep_assert_held(&cursor->man->bdev->lru_lock); + + list_move_tail(&cursor->hitch.link, &cursor->man->lru[cursor->priority]); + ttm_resource_cursor_clear_bulk(cursor); +} + /** * ttm_lru_first_res_or_null() - Return the first resource on an lru list * @head: The list head of the lru list. diff --git a/include/drm/ttm/ttm_bo.h b/include/drm/ttm/ttm_bo.h index 32791c4db2a9..e430b643d7f4 100644 --- a/include/drm/ttm/ttm_bo.h +++ b/include/drm/ttm/ttm_bo.h @@ -513,6 +513,8 @@ struct ttm_bo_lru_cursor { void ttm_bo_lru_cursor_fini(struct ttm_bo_lru_cursor *curs); +void ttm_bo_lru_cursor_skip_priority(struct ttm_bo_lru_cursor *curs); + struct ttm_bo_lru_cursor * ttm_bo_lru_cursor_init(struct ttm_bo_lru_cursor *curs, struct ttm_resource_manager *man, diff --git a/include/drm/ttm/ttm_resource.h b/include/drm/ttm/ttm_resource.h index 714c533622c2..d90e1cffa862 100644 --- a/include/drm/ttm/ttm_resource.h +++ b/include/drm/ttm/ttm_resource.h @@ -499,6 +499,7 @@ struct ttm_resource * ttm_resource_manager_first(struct ttm_resource_cursor *cursor); struct ttm_resource * ttm_resource_manager_next(struct ttm_resource_cursor *cursor); +void ttm_resource_cursor_skip_priority(struct ttm_resource_cursor *cursor); struct ttm_resource * ttm_lru_first_res_or_null(struct list_head *head); -- 2.34.1
