Queue-scoped EVENTFD subscriptions originally used queue_id as part of the routing key. However, queue_id is only a UAPI-visible handle and can be reused after a queue is destroyed, making it unsuitable as a stable identifier for internal EVENTFD tracking.
Rework queue-scoped EVENTFD handling to resolve queue_id to the corresponding amdgpu_usermode_queue object during bind and unbind. EVENTFD subscriptions now hold references to the actual queue objects instead of the reusable queue identifiers. Use the existing user queue refcounting infrastructure to keep queues alive while subscriptions exist, and release those references during unbind, manager teardown, and explicit queue cleanup. Introduce amdgpu_eventfd_remove_queue() to remove all subscriptions associated with a queue when that queue is being released. This ensures that EVENTFD does not retain stale queue references after queues are removed from the USERQ manager. Queue-scoped subscriptions are now matched using the queue pointer, while GPU-scoped subscriptions continue to operate without an associated queue. Also update the EVENTFD infrastructure to: distinguish queue-scoped and GPU-scoped event types, allow eventfd file descriptor 0 by rejecting only negative values, validate supported event types, and avoid relying on reusable queue identifiers for signaling. EVENTFD remains notification-only and does not carry event payloads. Cc: Alex Deucher <[email protected]> Suggested-by: Christian König <[email protected]> Signed-off-by: Srinivasan Shanmugam <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_eventfd.c | 240 ++++++++++++++------ drivers/gpu/drm/amd/amdgpu/amdgpu_eventfd.h | 19 +- drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 24 +- drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 4 + 4 files changed, 209 insertions(+), 78 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eventfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eventfd.c index 3a6e08a3d0c1..db743435605f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eventfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eventfd.c @@ -24,37 +24,63 @@ /* * Render-node eventfd subscription infrastructure. * - * This module provides a simple event notification mechanism for render-node - * clients using Linux eventfd objects. + * EVENTFD is notification-only. It wakes userspace when a GPU event happens. + * Event metadata/details are expected to be consumed separately through the + * corresponding wait/event path. * - * Userspace can bind an eventfd to a userspace-defined event_id. When the - * driver signals that event_id, all eventfds bound to it are notified. - * - * This mechanism is intended to support lightweight GPU event notifications - * without polling from userspace. + * Queue-scoped subscriptions use queue_id only for lookup at bind/unbind time. + * The EVENTFD entry stores a refcounted queue pointer, not the reusable UAPI + * queue_id. */ #include <linux/slab.h> #include <linux/err.h> +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" #include "amdgpu_eventfd.h" +#include "amdgpu_userq.h" #define AMDGPU_EVENTFD_MAX_BINDS 4096 +static bool amdgpu_eventfd_valid_type(u32 event_type) +{ + switch (event_type) { + case DRM_AMDGPU_EVENT_TYPE_USERQ_EOP: + case DRM_AMDGPU_EVENT_TYPE_QUEUE_RESET: + case DRM_AMDGPU_EVENT_TYPE_MEMORY_EXCEPTION: + case DRM_AMDGPU_EVENT_TYPE_SCRATCH: + case DRM_AMDGPU_EVENT_TYPE_GPU_RESET: + return true; + default: + return false; + } +} + +static bool amdgpu_eventfd_queue_scoped(u32 event_type) +{ + switch (event_type) { + case DRM_AMDGPU_EVENT_TYPE_USERQ_EOP: + case DRM_AMDGPU_EVENT_TYPE_QUEUE_RESET: + case DRM_AMDGPU_EVENT_TYPE_SCRATCH: + return true; + default: + return false; + } +} + /** - * amdgpu_eventfd_id_alloc - allocate an event id container - * @event_id: userspace-defined event identifier - * - * Each event_id represents a notification category. Multiple eventfds can - * be bound to the same event_id. + * amdgpu_eventfd_id_alloc - allocate an event type container + * @event_type: kernel-defined AMDGPU event type * - * This function allocates the container which stores the list of eventfds - * associated with that event_id. + * Each event_type has one container. For queue-scoped events, individual + * subscriptions inside the container are distinguished by the refcounted + * queue pointer stored in each entry. * * Return: * Pointer to the newly allocated structure or NULL on failure. */ -static struct amdgpu_eventfd_id *amdgpu_eventfd_id_alloc(u32 event_id) +static struct amdgpu_eventfd_id *amdgpu_eventfd_id_alloc(u32 event_type) { struct amdgpu_eventfd_id *id; @@ -62,43 +88,40 @@ static struct amdgpu_eventfd_id *amdgpu_eventfd_id_alloc(u32 event_id) if (!id) return NULL; - id->event_id = event_id; + id->event_type = event_type; INIT_HLIST_HEAD(&id->entries); id->n_entries = 0; + return id; } /** - * amdgpu_eventfd_id_get_or_create - find or create an event_id entry + * amdgpu_eventfd_id_get_or_create - find or create an event_type entry * @mgr: eventfd manager - * @event_id: event identifier - * - * This helper returns the container associated with the given event_id. - * If it does not exist, it will create one. + * @event_type: kernel-defined AMDGPU event type * - * The function is designed to be callable without holding any locks. - * Memory allocation is done outside the xarray lock to avoid blocking - * inside critical sections. + * This helper returns the container associated with the given event_type. + * If it does not exist, it creates one. * * Return: - * Pointer to the event_id structure or NULL on failure. + * Pointer to the event_type structure or NULL on failure. */ static struct amdgpu_eventfd_id * -amdgpu_eventfd_id_get_or_create(struct amdgpu_eventfd_mgr *mgr, u32 event_id) +amdgpu_eventfd_id_get_or_create(struct amdgpu_eventfd_mgr *mgr, u32 event_type) { struct amdgpu_eventfd_id *id; struct amdgpu_eventfd_id *new_id; - XA_STATE(xas, &mgr->ids, event_id); + XA_STATE(xas, &mgr->ids, event_type); unsigned long flags; int r; xa_lock_irqsave(&mgr->ids, flags); - id = xa_load(&mgr->ids, event_id); + id = xa_load(&mgr->ids, event_type); xa_unlock_irqrestore(&mgr->ids, flags); if (id) return id; - new_id = amdgpu_eventfd_id_alloc(event_id); + new_id = amdgpu_eventfd_id_alloc(event_type); if (!new_id) return NULL; @@ -174,6 +197,7 @@ void amdgpu_eventfd_mgr_fini(struct amdgpu_eventfd_mgr *mgr) hlist_for_each_entry_safe(e, tmp, &id->entries, hnode) { hlist_del(&e->hnode); + amdgpu_userq_put(e->queue); eventfd_ctx_put(e->ctx); kfree(e); } @@ -186,51 +210,61 @@ void amdgpu_eventfd_mgr_fini(struct amdgpu_eventfd_mgr *mgr) } /** - * amdgpu_eventfd_bind - bind eventfd to an event_id + * amdgpu_eventfd_bind - bind eventfd to an EVENTFD subscription * @mgr: eventfd manager - * @event_id: userspace event identifier + * @userq_mgr: user queue manager used to resolve queue_id + * @event_type: kernel-defined AMDGPU event type + * @queue_id: UAPI queue id for queue-scoped events, or 0 for GPU-scoped events * @eventfd: eventfd file descriptor * - * This function allows userspace to subscribe to notifications for a - * specific event_id. - * - * Multiple eventfds can be bound to the same event_id. - * - * Duplicate bindings of the same eventfd are treated as success and do - * not create additional entries. + * For queue-scoped events, queue_id is used only to look up the queue. + * The entry stores the refcounted queue pointer, not queue_id. * * Return: * 0 on success, negative error code on failure. */ -int amdgpu_eventfd_bind(struct amdgpu_eventfd_mgr *mgr, u32 event_id, int eventfd) +int amdgpu_eventfd_bind(struct amdgpu_eventfd_mgr *mgr, + struct amdgpu_userq_mgr *userq_mgr, + u32 event_type, u32 queue_id, int eventfd) { struct amdgpu_eventfd_id *id; struct amdgpu_eventfd_entry *e, *it; struct eventfd_ctx *ctx; + struct amdgpu_usermode_queue *queue = NULL; unsigned long flags; bool dup = false; - if (!mgr || !event_id || eventfd < 0) + if (!mgr || eventfd < 0 || !amdgpu_eventfd_valid_type(event_type)) return -EINVAL; - /* - * Enforce total bind limit without a separate manager lock. - * For duplicate binds, we decrement back before returning success. - */ + if (amdgpu_eventfd_queue_scoped(event_type)) { + if (!userq_mgr || !queue_id) + return -EINVAL; + + queue = amdgpu_userq_get(userq_mgr, queue_id); + if (!queue) + return -ENOENT; + } else if (queue_id) { + return -EINVAL; + } + if (atomic_inc_return(&mgr->bind_count) > AMDGPU_EVENTFD_MAX_BINDS) { atomic_dec(&mgr->bind_count); + amdgpu_userq_put(queue); return -ENOSPC; } ctx = eventfd_ctx_fdget(eventfd); if (IS_ERR(ctx)) { atomic_dec(&mgr->bind_count); + amdgpu_userq_put(queue); return PTR_ERR(ctx); } - id = amdgpu_eventfd_id_get_or_create(mgr, event_id); + id = amdgpu_eventfd_id_get_or_create(mgr, event_type); if (!id) { eventfd_ctx_put(ctx); + amdgpu_userq_put(queue); atomic_dec(&mgr->bind_count); return -ENOMEM; } @@ -238,7 +272,7 @@ int amdgpu_eventfd_bind(struct amdgpu_eventfd_mgr *mgr, u32 event_id, int eventf /* check for duplicate binding */ xa_lock_irqsave(&mgr->ids, flags); hlist_for_each_entry(it, &id->entries, hnode) { - if (it->ctx == ctx) { + if (it->ctx == ctx && it->queue == queue) { dup = true; break; } @@ -247,6 +281,7 @@ int amdgpu_eventfd_bind(struct amdgpu_eventfd_mgr *mgr, u32 event_id, int eventf if (dup) { eventfd_ctx_put(ctx); + amdgpu_userq_put(queue); atomic_dec(&mgr->bind_count); return 0; } @@ -255,10 +290,13 @@ int amdgpu_eventfd_bind(struct amdgpu_eventfd_mgr *mgr, u32 event_id, int eventf e = kzalloc(sizeof(*e), GFP_KERNEL); if (!e) { eventfd_ctx_put(ctx); + amdgpu_userq_put(queue); atomic_dec(&mgr->bind_count); return -ENOMEM; } + e->ctx = ctx; + e->queue = queue; /* * Re-check duplicate under lock to close the race with another bind() @@ -266,7 +304,7 @@ int amdgpu_eventfd_bind(struct amdgpu_eventfd_mgr *mgr, u32 event_id, int eventf */ xa_lock_irqsave(&mgr->ids, flags); hlist_for_each_entry(it, &id->entries, hnode) { - if (it->ctx == ctx) { + if (it->ctx == ctx && it->queue == queue) { dup = true; break; } @@ -281,6 +319,7 @@ int amdgpu_eventfd_bind(struct amdgpu_eventfd_mgr *mgr, u32 event_id, int eventf if (dup) { eventfd_ctx_put(ctx); + amdgpu_userq_put(queue); kfree(e); atomic_dec(&mgr->bind_count); return 0; @@ -290,53 +329,70 @@ int amdgpu_eventfd_bind(struct amdgpu_eventfd_mgr *mgr, u32 event_id, int eventf } /** - * amdgpu_eventfd_unbind - remove eventfd binding + * amdgpu_eventfd_unbind - remove EVENTFD binding * @mgr: eventfd manager - * @event_id: event identifier + * @userq_mgr: user queue manager used to resolve queue_id + * @event_type: kernel-defined AMDGPU event type + * @queue_id: UAPI queue id for queue-scoped events, or 0 for GPU-scoped events * @eventfd: eventfd file descriptor * - * Removes an existing binding between an event_id and an eventfd. - * * Return: * 0 if removed, -ENOENT if binding does not exist. */ -int amdgpu_eventfd_unbind(struct amdgpu_eventfd_mgr *mgr, u32 event_id, int eventfd) +int amdgpu_eventfd_unbind(struct amdgpu_eventfd_mgr *mgr, + struct amdgpu_userq_mgr *userq_mgr, + u32 event_type, u32 queue_id, int eventfd) { struct amdgpu_eventfd_id *id; struct amdgpu_eventfd_entry *e; struct hlist_node *tmp; struct eventfd_ctx *ctx; + struct amdgpu_usermode_queue *queue = NULL; unsigned long flags; bool removed = false; - if (!mgr || !event_id || eventfd < 0) + if (!mgr || eventfd < 0 || !amdgpu_eventfd_valid_type(event_type)) return -EINVAL; + if (amdgpu_eventfd_queue_scoped(event_type)) { + if (!userq_mgr || !queue_id) + return -EINVAL; + + queue = amdgpu_userq_get(userq_mgr, queue_id); + if (!queue) + return -ENOENT; + } else if (queue_id) { + return -EINVAL; + } + ctx = eventfd_ctx_fdget(eventfd); - if (IS_ERR(ctx)) + if (IS_ERR(ctx)) { + amdgpu_userq_put(queue); return PTR_ERR(ctx); + } xa_lock_irqsave(&mgr->ids, flags); - id = xa_load(&mgr->ids, event_id); + id = xa_load(&mgr->ids, event_type); if (!id) goto out_unlock; hlist_for_each_entry_safe(e, tmp, &id->entries, hnode) { - if (e->ctx != ctx) + if (e->ctx != ctx || e->queue != queue) continue; hlist_del(&e->hnode); id->n_entries--; removed = true; + amdgpu_userq_put(e->queue); eventfd_ctx_put(e->ctx); kfree(e); atomic_dec(&mgr->bind_count); if (!id->n_entries) { - __xa_erase(&mgr->ids, event_id); + __xa_erase(&mgr->ids, event_type); kfree(id); } @@ -346,27 +402,75 @@ int amdgpu_eventfd_unbind(struct amdgpu_eventfd_mgr *mgr, u32 event_id, int even out_unlock: xa_unlock_irqrestore(&mgr->ids, flags); eventfd_ctx_put(ctx); + amdgpu_userq_put(queue); return removed ? 0 : -ENOENT; } /** - * amdgpu_eventfd_signal - notify all eventfds bound to event_id + * amdgpu_eventfd_remove_queue - remove all EVENTFD bindings for a queue * @mgr: eventfd manager - * @event_id: event identifier + * @queue: queue being destroyed/released * - * This function is typically called from interrupt context. + * Remove all subscriptions that hold a reference to @queue. + * This is called when the queue id is released so EVENTFD can drop + * its queue references before the queue is finally destroyed. + */ +void amdgpu_eventfd_remove_queue(struct amdgpu_eventfd_mgr *mgr, + struct amdgpu_usermode_queue *queue) +{ + struct amdgpu_eventfd_id *id; + struct amdgpu_eventfd_entry *e; + struct hlist_node *tmp; + unsigned long index; + unsigned long flags; + + if (!mgr || !queue) + return; + + xa_lock_irqsave(&mgr->ids, flags); + + xa_for_each(&mgr->ids, index, id) { + hlist_for_each_entry_safe(e, tmp, &id->entries, hnode) { + if (e->queue != queue) + continue; + + hlist_del(&e->hnode); + id->n_entries--; + + eventfd_ctx_put(e->ctx); + amdgpu_userq_put(e->queue); + kfree(e); + + atomic_dec(&mgr->bind_count); + } + + if (!id->n_entries) { + __xa_erase(&mgr->ids, index); + kfree(id); + } + } + + xa_unlock_irqrestore(&mgr->ids, flags); +} + +/** + * amdgpu_eventfd_signal - notify all matching eventfd subscriptions + * @mgr: eventfd manager + * @event_type: kernel-defined AMDGPU event type + * @queue: queue pointer for queue-scoped events, or NULL for GPU-scoped events * - * All eventfds registered for the given event_id will be signaled. - * Userspace processes waiting on those eventfds will wake up. + * This can run from IRQ context. The queue pointer must refer to the actual + * queue object, not the reusable UAPI queue_id. */ -void amdgpu_eventfd_signal(struct amdgpu_eventfd_mgr *mgr, u32 event_id) +void amdgpu_eventfd_signal(struct amdgpu_eventfd_mgr *mgr, u32 event_type, + struct amdgpu_usermode_queue *queue) { struct amdgpu_eventfd_id *id; struct amdgpu_eventfd_entry *e; unsigned long flags; - if (!mgr || !event_id) + if (!mgr || !amdgpu_eventfd_valid_type(event_type)) return; /* @@ -375,10 +479,12 @@ void amdgpu_eventfd_signal(struct amdgpu_eventfd_mgr *mgr, u32 event_id) */ xa_lock_irqsave(&mgr->ids, flags); - id = xa_load(&mgr->ids, event_id); + id = xa_load(&mgr->ids, event_type); if (id) { - hlist_for_each_entry(e, &id->entries, hnode) - eventfd_signal(e->ctx); + hlist_for_each_entry(e, &id->entries, hnode) { + if (e->queue == queue) + eventfd_signal(e->ctx); + } } xa_unlock_irqrestore(&mgr->ids, flags); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eventfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_eventfd.h index 248afb1f2f14..9ea3283e92bd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eventfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eventfd.h @@ -32,13 +32,17 @@ #include <linux/xarray.h> #include <linux/atomic.h> +struct amdgpu_userq_mgr; +struct amdgpu_usermode_queue; + struct amdgpu_eventfd_entry { struct eventfd_ctx *ctx; + struct amdgpu_usermode_queue *queue; struct hlist_node hnode; }; struct amdgpu_eventfd_id { - u32 event_id; + u32 event_type; struct hlist_head entries; u32 n_entries; }; @@ -51,9 +55,16 @@ struct amdgpu_eventfd_mgr { void amdgpu_eventfd_mgr_init(struct amdgpu_eventfd_mgr *mgr); void amdgpu_eventfd_mgr_fini(struct amdgpu_eventfd_mgr *mgr); -int amdgpu_eventfd_bind(struct amdgpu_eventfd_mgr *mgr, u32 event_id, int eventfd); -int amdgpu_eventfd_unbind(struct amdgpu_eventfd_mgr *mgr, u32 event_id, int eventfd); +int amdgpu_eventfd_bind(struct amdgpu_eventfd_mgr *mgr, + struct amdgpu_userq_mgr *userq_mgr, + u32 event_type, u32 queue_id, int eventfd); +int amdgpu_eventfd_unbind(struct amdgpu_eventfd_mgr *mgr, + struct amdgpu_userq_mgr *userq_mgr, + u32 event_type, u32 queue_id, int eventfd); +void amdgpu_eventfd_remove_queue(struct amdgpu_eventfd_mgr *mgr, + struct amdgpu_usermode_queue *queue); -void amdgpu_eventfd_signal(struct amdgpu_eventfd_mgr *mgr, u32 event_id); +void amdgpu_eventfd_signal(struct amdgpu_eventfd_mgr *mgr, u32 event_type, + struct amdgpu_usermode_queue *queue); #endif /* __AMDGPU_EVENTFD_H__ */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index f7c750094393..0db128def289 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -649,22 +649,32 @@ int amdgpu_eventfd_ioctl(struct drm_device *dev, void *data, if (args->flags || !args->event_type || args->eventfd < 0) return -EINVAL; - /* - * Queue-scoped subscriptions are enabled by the later queue-reference - * routing patch. Until then, keep queue_id zero. - */ - if (args->queue_id) + switch (args->event_type) { + case DRM_AMDGPU_EVENT_TYPE_USERQ_EOP: + case DRM_AMDGPU_EVENT_TYPE_QUEUE_RESET: + case DRM_AMDGPU_EVENT_TYPE_SCRATCH: + break; + case DRM_AMDGPU_EVENT_TYPE_MEMORY_EXCEPTION: + if (args->queue_id) + return -EINVAL; + break; + default: return -EINVAL; + } switch (args->op) { case DRM_AMDGPU_EVENTFD_OP_BIND: return amdgpu_eventfd_bind(&fpriv->eventfd_mgr, + &fpriv->userq_mgr, args->event_type, + args->queue_id, args->eventfd); case DRM_AMDGPU_EVENTFD_OP_UNBIND: return amdgpu_eventfd_unbind(&fpriv->eventfd_mgr, - args->event_type, - args->eventfd); + &fpriv->userq_mgr, + args->event_type, + args->queue_id, + args->eventfd); default: return -EINVAL; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c index 376813e9623f..45981adbd7d3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c @@ -869,6 +869,8 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data, if (!queue) return -ENOENT; + amdgpu_eventfd_remove_queue(&fpriv->eventfd_mgr, queue); + amdgpu_userq_put(queue); break; } @@ -1229,6 +1231,8 @@ void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr) if (!queue) break; + amdgpu_eventfd_remove_queue(amdgpu_userq_eventfd_mgr(userq_mgr), queue); + amdgpu_userq_put(queue); } -- 2.34.1
