Add the per-file WAIT_EVENT manager and lifetime plumbing used to store and retrieve event records for render-node clients.
The manager maintains a list of pending WAIT_EVENT records, supports blocking waits from userspace, copies the first matching event record to userspace, and enforces single-consumer semantics by removing records once they are consumed. Register the WAIT_EVENT ioctl so render-node clients can access the per-file WAIT_EVENT manager. For queue-scoped events, queue_id is resolved to the corresponding usermode queue object at the ioctl boundary. Pending records are matched using queue pointer equality rather than queue_id values, avoiding internal routing through reusable userspace handles. Pending WAIT_EVENT records hold queue references while queued. Those references are released when records are consumed, explicitly removed, or destroyed during manager teardown. Embed the WAIT_EVENT manager in amdgpu_fpriv and tie its lifetime to drm_file. Initialize the manager during file open and destroy it during file close so that pending records are cleaned up and blocked waiters are released before file-private state is freed. Changes since v8: - Squashed "Register WAIT_EVENT ioctl" into "Add wait-event manager and per-file lifetime plumbing", as suggested by Alex. Cc: Alex Deucher <[email protected]> Cc: Christian König <[email protected]> Signed-off-by: Srinivasan Shanmugam <[email protected]> --- drivers/gpu/drm/amd/amdgpu/Makefile | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 5 +- drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 5 +- .../gpu/drm/amd/amdgpu/amdgpu_wait_event.c | 286 ++++++++++++++++++ .../gpu/drm/amd/amdgpu/amdgpu_wait_event.h | 74 +++++ 6 files changed, 370 insertions(+), 3 deletions(-) create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_wait_event.c create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_wait_event.h diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index b7897f98436c..c4cee6a6bc64 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -72,7 +72,7 @@ amdgpu-y += amdgpu_device.o amdgpu_reg_access.o amdgpu_doorbell_mgr.o amdgpu_kms amdgpu_eeprom.o amdgpu_mca.o amdgpu_psp_ta.o amdgpu_lsdma.o amdgpu_lockdep.o \ amdgpu_ring_mux.o amdgpu_xcp.o amdgpu_seq64.o amdgpu_dev_coredump.o \ amdgpu_cper.o amdgpu_userq_fence.o amdgpu_eviction_fence.o amdgpu_ip.o \ - amdgpu_wb.o amdgpu_cwsr.o amdgpu_events.o amdgpu_eventfd.o + amdgpu_wb.o amdgpu_cwsr.o amdgpu_events.o amdgpu_eventfd.o amdgpu_wait_event.o amdgpu-$(CONFIG_PROC_FS) += amdgpu_fdinfo.o diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 4f42888e2647..91f1dc737aee 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -104,6 +104,7 @@ #include "amdgpu_fdinfo.h" #include "amdgpu_mca.h" #include "amdgpu_eventfd.h" +#include "amdgpu_wait_event.h" #include "amdgpu_ras.h" #include "amdgpu_lockdep.h" #include "amdgpu_cper.h" @@ -427,13 +428,15 @@ struct amdgpu_fpriv { uint32_t xcp_id; struct amdgpu_eventfd_mgr eventfd_mgr; + struct amdgpu_wait_event_mgr wait_event_mgr; }; struct drm_device; struct drm_file; int amdgpu_eventfd_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); - +int amdgpu_wait_event_drm_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv); int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv); /* diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index e90cf67c1cd8..b738a1bdf9d3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -3097,6 +3097,7 @@ const struct drm_ioctl_desc amdgpu_ioctls_kms[] = { DRM_IOCTL_DEF_DRV(AMDGPU_GEM_LIST_HANDLES, amdgpu_gem_list_handles_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(AMDGPU_PROC_OPTIONS, amdgpu_proc_options_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(AMDGPU_EVENTFD, amdgpu_eventfd_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(AMDGPU_WAIT_EVENT, amdgpu_wait_event_drm_ioctl, DRM_RENDER_ALLOW), }; static const struct drm_driver amdgpu_kms_driver = { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index ccc9c3f8aba7..72bea83d7408 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -1681,6 +1681,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv) } amdgpu_eventfd_mgr_init(&fpriv->eventfd_mgr); + amdgpu_wait_event_mgr_init(&fpriv->wait_event_mgr); pasid = amdgpu_pasid_alloc(16); if (pasid < 0) { @@ -1754,6 +1755,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv) if (pasid) amdgpu_pasid_free(pasid); + amdgpu_wait_event_mgr_fini(&fpriv->wait_event_mgr); kfree(fpriv); out_suspend: @@ -1784,8 +1786,9 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev, if (!fpriv) return; - /* Drop all subscriptions before fpriv goes away. */ + /* Drop eventfd subscriptions and pending wait-event records. */ amdgpu_eventfd_mgr_fini(&fpriv->eventfd_mgr); + amdgpu_wait_event_mgr_fini(&fpriv->wait_event_mgr); pm_runtime_get_sync(dev->dev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_wait_event.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_wait_event.c new file mode 100644 index 000000000000..f98de1d94b56 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_wait_event.c @@ -0,0 +1,286 @@ +/* + * Copyright 2026 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <linux/jiffies.h> +#include <linux/sched/signal.h> +#include <linux/slab.h> +#include <linux/uaccess.h> + +#include "amdgpu.h" +#include "amdgpu_userq_internal.h" +#include "amdgpu_wait_event.h" + +static long amdgpu_wait_event_to_jiffies(__s64 timeout_ns) +{ + unsigned long long t; + long timeout; + + if (timeout_ns < 0) + return MAX_SCHEDULE_TIMEOUT; + if (timeout_ns == 0) + return 0; + + t = nsecs_to_jiffies(timeout_ns); + if (t > MAX_SCHEDULE_TIMEOUT) + timeout = MAX_SCHEDULE_TIMEOUT - 1; + else + timeout = t; + + return timeout ?: 1; +} + +static bool amdgpu_wait_event_valid_type(u32 event_type) +{ + switch (event_type) { + case DRM_AMDGPU_EVENT_TYPE_USERQ_EOP: + case DRM_AMDGPU_EVENT_TYPE_QUEUE_RESET: + case DRM_AMDGPU_EVENT_TYPE_MEMORY_EXCEPTION: + case DRM_AMDGPU_EVENT_TYPE_SCRATCH: + case DRM_AMDGPU_EVENT_TYPE_GPU_RESET: + return true; + default: + return false; + } +} + +static void +amdgpu_wait_event_record_free(struct amdgpu_wait_event_record *rec) +{ + if (!rec) + return; + + if (rec->queue) + amdgpu_userq_put(rec->queue); + + kfree(rec); +} + +static bool +amdgpu_wait_event_match(const struct drm_amdgpu_wait_event *args, + struct amdgpu_usermode_queue *queue, + const struct amdgpu_wait_event_record *rec) +{ + if (rec->data.event_type != args->event_type) + return false; + + if (amdgpu_wait_event_type_is_queue_scoped(args->event_type)) + return rec->queue == queue; + + return !queue; +} + +static bool +amdgpu_wait_event_has_match(struct amdgpu_wait_event_mgr *mgr, + const struct drm_amdgpu_wait_event *args, + struct amdgpu_usermode_queue *queue) +{ + struct amdgpu_wait_event_record *rec; + bool found = false; + unsigned long flags; + + spin_lock_irqsave(&mgr->lock, flags); + list_for_each_entry(rec, &mgr->pending, node) { + if (amdgpu_wait_event_match(args, queue, rec)) { + found = true; + break; + } + } + spin_unlock_irqrestore(&mgr->lock, flags); + + return found; +} + +static struct amdgpu_wait_event_record * +amdgpu_wait_event_pop_match(struct amdgpu_wait_event_mgr *mgr, + const struct drm_amdgpu_wait_event *args, + struct amdgpu_usermode_queue *queue) +{ + struct amdgpu_wait_event_record *rec, *tmp, *found = NULL; + unsigned long flags; + + spin_lock_irqsave(&mgr->lock, flags); + list_for_each_entry_safe(rec, tmp, &mgr->pending, node) { + if (amdgpu_wait_event_match(args, queue, rec)) { + list_del(&rec->node); + found = rec; + break; + } + } + spin_unlock_irqrestore(&mgr->lock, flags); + + return found; +} + +static int +amdgpu_wait_event_get_queue(struct amdgpu_fpriv *fpriv, + const struct drm_amdgpu_wait_event *args, + struct amdgpu_usermode_queue **queue) +{ + *queue = NULL; + + if (!amdgpu_wait_event_valid_type(args->event_type)) + return -EINVAL; + + if (amdgpu_wait_event_type_is_queue_scoped(args->event_type)) { + if (!args->queue_id) + return -EINVAL; + + *queue = amdgpu_userq_get(&fpriv->userq_mgr, args->queue_id); + if (!*queue) + return -ENOENT; + + return 0; + } + + if (args->queue_id) + return -EINVAL; + + return 0; +} + +void amdgpu_wait_event_mgr_init(struct amdgpu_wait_event_mgr *mgr) +{ + spin_lock_init(&mgr->lock); + init_waitqueue_head(&mgr->wq); + INIT_LIST_HEAD(&mgr->pending); + atomic64_set(&mgr->seqno, 0); + mgr->dead = false; +} + +void amdgpu_wait_event_mgr_fini(struct amdgpu_wait_event_mgr *mgr) +{ + struct amdgpu_wait_event_record *rec, *tmp; + unsigned long flags; + LIST_HEAD(removed); + + spin_lock_irqsave(&mgr->lock, flags); + mgr->dead = true; + list_splice_init(&mgr->pending, &removed); + spin_unlock_irqrestore(&mgr->lock, flags); + + list_for_each_entry_safe(rec, tmp, &removed, node) { + list_del(&rec->node); + amdgpu_wait_event_record_free(rec); + } + + wake_up_interruptible_all(&mgr->wq); +} + +void amdgpu_wait_event_remove_queue(struct amdgpu_wait_event_mgr *mgr, + struct amdgpu_usermode_queue *queue) +{ + struct amdgpu_wait_event_record *rec, *tmp; + unsigned long flags; + LIST_HEAD(removed); + + if (!mgr || !queue) + return; + + spin_lock_irqsave(&mgr->lock, flags); + list_for_each_entry_safe(rec, tmp, &mgr->pending, node) { + if (rec->queue != queue) + continue; + + list_move_tail(&rec->node, &removed); + } + spin_unlock_irqrestore(&mgr->lock, flags); + + list_for_each_entry_safe(rec, tmp, &removed, node) { + list_del(&rec->node); + amdgpu_wait_event_record_free(rec); + } + + wake_up_interruptible_all(&mgr->wq); +} + +int amdgpu_wait_event_drm_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct amdgpu_fpriv *fpriv = file_priv->driver_priv; + struct drm_amdgpu_wait_event *args = data; + struct amdgpu_usermode_queue *queue = NULL; + struct amdgpu_wait_event_mgr *mgr; + struct amdgpu_wait_event_record *rec; + long timeout; + int ret; + + if (!fpriv) + return -EINVAL; + + if (args->flags || !args->event_type || !args->out_ptr) + return -EINVAL; + + if (args->out_size < sizeof(struct drm_amdgpu_wait_event_data)) + return -EINVAL; + + ret = amdgpu_wait_event_get_queue(fpriv, args, &queue); + if (ret) + return ret; + + mgr = &fpriv->wait_event_mgr; + timeout = amdgpu_wait_event_to_jiffies(args->timeout_ns); + + for (;;) { + rec = amdgpu_wait_event_pop_match(mgr, args, queue); + if (rec) + break; + + if (READ_ONCE(mgr->dead)) { + ret = -EIO; + goto out_put_queue; + } + + if (signal_pending(current)) { + ret = -ERESTARTSYS; + goto out_put_queue; + } + + if (!timeout) { + ret = -ETIME; + goto out_put_queue; + } + + timeout = wait_event_interruptible_timeout(mgr->wq, + READ_ONCE(mgr->dead) || + amdgpu_wait_event_has_match(mgr, args, queue), + timeout); + if (timeout < 0) { + ret = timeout; + goto out_put_queue; + } + } + + if (copy_to_user(u64_to_user_ptr(args->out_ptr), &rec->data, + sizeof(rec->data))) + ret = -EFAULT; + else + ret = 0; + + amdgpu_wait_event_record_free(rec); + +out_put_queue: + if (queue) + amdgpu_userq_put(queue); + + return ret; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_wait_event.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_wait_event.h new file mode 100644 index 000000000000..e887fffdc4ac --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_wait_event.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright 2026 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_WAIT_EVENT_H__ +#define __AMDGPU_WAIT_EVENT_H__ + +#include <linux/list.h> +#include <linux/spinlock.h> +#include <linux/wait.h> +#include <linux/atomic.h> +#include <drm/drm_device.h> +#include <drm/drm_file.h> +#include <uapi/drm/amdgpu_drm.h> + +struct amdgpu_usermode_queue; + +struct amdgpu_wait_event_record { + struct list_head node; + struct amdgpu_usermode_queue *queue; + struct drm_amdgpu_wait_event_data data; +}; + +struct amdgpu_wait_event_mgr { + /* Used when adding, removing, or checking pending events. */ + spinlock_t lock; + wait_queue_head_t wq; + struct list_head pending; + atomic64_t seqno; + bool dead; +}; + +void amdgpu_wait_event_mgr_init(struct amdgpu_wait_event_mgr *mgr); +void amdgpu_wait_event_mgr_fini(struct amdgpu_wait_event_mgr *mgr); + +void amdgpu_wait_event_remove_queue(struct amdgpu_wait_event_mgr *mgr, + struct amdgpu_usermode_queue *queue); + +int amdgpu_wait_event_drm_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv); + +static inline bool amdgpu_wait_event_type_is_queue_scoped(u32 event_type) +{ + switch (event_type) { + case DRM_AMDGPU_EVENT_TYPE_USERQ_EOP: + case DRM_AMDGPU_EVENT_TYPE_QUEUE_RESET: + case DRM_AMDGPU_EVENT_TYPE_SCRATCH: + return true; + default: + return false; + } +} + +#endif /* __AMDGPU_WAIT_EVENT_H__ */ -- 2.34.1
