Module: Mesa Branch: main Commit: 6cc42bc0e77c7fb920246543fb29fec0f998aa62 URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=6cc42bc0e77c7fb920246543fb29fec0f998aa62
Author: Jason Volk <[email protected]> Date: Wed Mar 9 12:34:08 2022 -0800 r600: Fix userspace pointer support for evergreen compute. Resources returned by r600_buffer_from_user_memory() are not compatible with the evergreen compute memory pool, though they're added to it anyway. This results in a segfault reproducible from Clover when the user passes CL_MEM_USE_HOST_PTR. This patch allows user_ptr resources to participate in the compute global memory pool as intended. The result appears to finally allow for zero-copy DMA out of userspace for anonymous pages. Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/16114> --- src/gallium/drivers/r600/compute_memory_pool.c | 44 +++++++++++--------------- src/gallium/drivers/r600/compute_memory_pool.h | 6 ++++ src/gallium/drivers/r600/evergreen_compute.c | 10 ++++-- src/gallium/drivers/r600/r600_buffer_common.c | 17 ++++++++-- 4 files changed, 47 insertions(+), 30 deletions(-) diff --git a/src/gallium/drivers/r600/compute_memory_pool.c b/src/gallium/drivers/r600/compute_memory_pool.c index 03b0cc524c1..d0f6979dc66 100644 --- a/src/gallium/drivers/r600/compute_memory_pool.c +++ b/src/gallium/drivers/r600/compute_memory_pool.c @@ -350,7 +350,7 @@ static int compute_memory_promote_item(struct compute_memory_pool *pool, * In this case, we need to keep the temporary buffer 'alive' * because it is possible to keep a map active for reading * while a kernel (that reads from it) executes */ - if (!(item->status & ITEM_MAPPED_FOR_READING)) { + if (!(item->status & ITEM_MAPPED_FOR_READING) && !is_item_user_ptr(item)) { pool->screen->b.b.resource_destroy(screen, src); item->real_buffer = NULL; } @@ -494,6 +494,22 @@ static void compute_memory_move_item(struct compute_memory_pool *pool, item->start_in_dw = new_start_in_dw; } +/** + * Frees one item for compute_memory_free() + */ +static void compute_memory_free_item(struct pipe_screen *screen, + struct compute_memory_item *item) +{ + struct pipe_resource *res = (struct pipe_resource *)item->real_buffer; + + list_del(&item->link); + + if (res && !is_item_user_ptr(item)) + screen->resource_destroy(screen, res); + + free(item); +} + /** * Frees the memory associated to the item with id \a id from the pool. * \param id The id of the item to be freed. @@ -502,45 +518,23 @@ void compute_memory_free(struct compute_memory_pool* pool, int64_t id) { struct compute_memory_item *item, *next; struct pipe_screen *screen = (struct pipe_screen *)pool->screen; - struct pipe_resource *res; COMPUTE_DBG(pool->screen, "* compute_memory_free() id + %"PRIi64" \n", id); LIST_FOR_EACH_ENTRY_SAFE(item, next, pool->item_list, link) { - if (item->id == id) { - if (item->link.next != pool->item_list) { pool->status |= POOL_FRAGMENTED; } - list_del(&item->link); - - if (item->real_buffer) { - res = (struct pipe_resource *)item->real_buffer; - pool->screen->b.b.resource_destroy( - screen, res); - } - - free(item); - + compute_memory_free_item(screen, item); return; } } LIST_FOR_EACH_ENTRY_SAFE(item, next, pool->unallocated_list, link) { - if (item->id == id) { - list_del(&item->link); - - if (item->real_buffer) { - res = (struct pipe_resource *)item->real_buffer; - pool->screen->b.b.resource_destroy( - screen, res); - } - - free(item); - + compute_memory_free_item(screen, item); return; } } diff --git a/src/gallium/drivers/r600/compute_memory_pool.h b/src/gallium/drivers/r600/compute_memory_pool.h index 3b90976276c..7ab1ad73b08 100644 --- a/src/gallium/drivers/r600/compute_memory_pool.h +++ b/src/gallium/drivers/r600/compute_memory_pool.h @@ -82,6 +82,12 @@ static inline int is_item_in_pool(struct compute_memory_item *item) return item->start_in_dw != -1; } +static inline int is_item_user_ptr(struct compute_memory_item *item) +{ + assert(item->real_buffer); + return item->real_buffer->b.is_user_ptr; +} + struct compute_memory_pool* compute_memory_pool_new(struct r600_screen *rscreen); void compute_memory_pool_delete(struct compute_memory_pool* pool); diff --git a/src/gallium/drivers/r600/evergreen_compute.c b/src/gallium/drivers/r600/evergreen_compute.c index f7eb788bfb4..4d10e2b604e 100644 --- a/src/gallium/drivers/r600/evergreen_compute.c +++ b/src/gallium/drivers/r600/evergreen_compute.c @@ -1277,6 +1277,9 @@ void *r600_compute_global_transfer_map(struct pipe_context *ctx, assert(box->y == 0); assert(box->z == 0); + if (buffer->base.b.is_user_ptr) + return NULL; + ///TODO: do it better, mapping is not possible if the pool is too big return pipe_buffer_map_range(ctx, dst, offset, box->width, usage, ptransfer); @@ -1311,9 +1314,12 @@ void r600_compute_global_buffer_destroy(struct pipe_screen *screen, rscreen = (struct r600_screen*)screen; compute_memory_free(rscreen->global_pool, buffer->chunk->id); - buffer->chunk = NULL; - free(res); + + if (buffer->base.b.is_user_ptr) + r600_buffer_destroy(screen, res); + else + free(res); } struct pipe_resource *r600_compute_global_buffer_create(struct pipe_screen *screen, diff --git a/src/gallium/drivers/r600/r600_buffer_common.c b/src/gallium/drivers/r600/r600_buffer_common.c index a47faafa928..d1e48c161cd 100644 --- a/src/gallium/drivers/r600/r600_buffer_common.c +++ b/src/gallium/drivers/r600/r600_buffer_common.c @@ -26,6 +26,7 @@ #include "r600_cs.h" #include "evergreen_compute.h" +#include "compute_memory_pool.h" #include "util/u_memory.h" #include "util/u_upload_mgr.h" #include <inttypes.h> @@ -347,7 +348,8 @@ void *r600_buffer_transfer_map(struct pipe_context *ctx, uint8_t *data; if (r600_resource(resource)->compute_global_bo) { - return r600_compute_global_transfer_map(ctx, resource, level, usage, box, ptransfer); + if ((data = r600_compute_global_transfer_map(ctx, resource, level, usage, box, ptransfer))) + return data; } assert(box->x + box->width <= resource->width0); @@ -524,8 +526,9 @@ void r600_buffer_transfer_unmap(struct pipe_context *ctx, { struct r600_common_context *rctx = (struct r600_common_context*)ctx; struct r600_transfer *rtransfer = (struct r600_transfer*)transfer; + struct r600_resource *rtransferr = r600_resource(transfer->resource); - if (r600_resource(transfer->resource)->compute_global_bo) { + if (rtransferr->compute_global_bo && !rtransferr->b.is_user_ptr) { r600_compute_global_transfer_unmap(ctx, transfer); return; } @@ -636,7 +639,15 @@ r600_buffer_from_user_memory(struct pipe_screen *screen, { struct r600_common_screen *rscreen = (struct r600_common_screen*)screen; struct radeon_winsys *ws = rscreen->ws; - struct r600_resource *rbuffer = r600_alloc_buffer_struct(screen, templ); + struct r600_resource *rbuffer; + + if ((templ->bind & PIPE_BIND_GLOBAL) && + (templ->bind & PIPE_BIND_COMPUTE_RESOURCE)) { + rbuffer = r600_resource(r600_compute_global_buffer_create(screen, templ)); + ((struct r600_resource_global *)rbuffer)->chunk->real_buffer = rbuffer; + } else { + rbuffer = r600_alloc_buffer_struct(screen, templ); + } rbuffer->domains = RADEON_DOMAIN_GTT; rbuffer->flags = 0;
