Implement the drain path that copies SPM counter records out of the per-XCC ring buffers into the destination buffer installed by AMDGPU_SPM_OP_SET_DEST_BUF; amdgpu_spm_work() was a stub until now.
The worker snapshots spm_cntr and lead_thread together under spm_irq_lock and bails out if either is NULL, so a work item still queued when RELEASE runs does not touch a dying session. It attaches the lead thread's mm with kthread_use_mm() so copy_to_user() targets the profiler's address space, and drains the XCC instances of the partition under spm_worker_mutex, which locks out SET_DEST_BUF and the RELEASE teardown. Per instance, amdgpu_spm_read_ring_buffer() snapshots is_spm_started and ring_rptr in one spm_irq_lock critical section, since _spm_stop() clears the flag and resets rptr under the same lock. The hardware write pointer is rejected unless it is below ring_size, because a corrupted value would make the copy run past the ring and leak kernel memory to userspace. Records are copied in one segment, or in two when the producer has wrapped, and the current rptr is always published back to the RLC with amdgpu_rlc_spm_set_rdptr(). Copies stop at the end of the user buffer and mark it filled, which sets are_users_buf_filled and wakes spm_buf_wq so a thread blocked in SET_DEST_BUF can install a new buffer. Nothing is propagated out of the worker; a failed copy, a bad wptr, or a missing or full user buffer set has_data_loss as a hint that the ring may have stalled. SET_DEST_BUF flushes the worker after installing a buffer unless every instance was started by that same call. The polling thread that schedules the work is added by a later patch. Signed-off-by: James Zhu <[email protected]> Reviewed-by: Bing Ma <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_spm.c | 226 +++++++++++++++++++++++- 1 file changed, 225 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_spm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_spm.c index b3132f9a8c0a..d17fb2963981 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_spm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_spm.c @@ -21,6 +21,8 @@ * OTHER DEALINGS IN THE SOFTWARE. */ +#include <linux/kthread.h> +#include <linux/sched/mm.h> #include <linux/sched/task.h> #include "amdgpu.h" @@ -49,9 +51,227 @@ static bool amdgpu_spm_is_owner(struct amdgpu_spm_mgr *spm_mgr, struct drm_file return spm_mgr->file == filp; } +static int amdgpu_spm_data_copy(struct amdgpu_spm_mgr *spm_mgr, + struct amdgpu_spm_cntr *spm_cntr, u32 ring_rptr, + u32 size_to_copy, int inst) +{ + struct amdgpu_spm_base *spm = &spm_cntr->spm[inst]; + u64 __user *user_address; + u64 *ring_buf; + u32 user_buf_space_left; + unsigned long flags; + u32 new_rptr; + + if (!spm->ubuf.user_addr) + return -EFAULT; + + user_address = (u64 __user *)((void __user *)spm->ubuf.user_addr + spm->size_copied); + /* From RLC spec, ring_rptr = 0 points to spm->cpu_addr + AMDGPU_SPM_META_SIZE */ + ring_buf = (u64 *)((u8 *)spm->cpu_addr + ring_rptr + AMDGPU_SPM_META_SIZE); + + user_buf_space_left = spm->ubuf.ubufsize - spm->size_copied; + + if (size_to_copy < user_buf_space_left) { + if (copy_to_user(user_address, ring_buf, size_to_copy)) { + spm->has_data_loss = true; + return -EFAULT; + } + spm->size_copied += size_to_copy; + new_rptr = ring_rptr + size_to_copy; + } else { + if (copy_to_user(user_address, ring_buf, user_buf_space_left)) { + spm->has_data_loss = true; + return -EFAULT; + } + + spm->size_copied = spm->ubuf.ubufsize; + new_rptr = ring_rptr + user_buf_space_left; + spm->is_user_buf_filled = true; + } + + /* _spm_stop() resets rptr to 0 and is serialized against the drain + * path by spm_worker_mutex, so this store cannot resurrect a stale + * rptr. spm_irq_lock pairs with the rptr snapshot taken under it. + */ + spin_lock_irqsave(&spm_mgr->spm_irq_lock, flags); + WRITE_ONCE(spm->ring_rptr, new_rptr); + spin_unlock_irqrestore(&spm_mgr->spm_irq_lock, flags); + + return 0; +} + +static int amdgpu_spm_read_ring_buffer(struct amdgpu_spm_mgr *spm_mgr, + struct amdgpu_spm_cntr *spm_cntr, int inst) +{ + struct amdgpu_profiler_xcp_mgr *prof_xcp_mgr = to_prof_xcp_mgr(spm_mgr, spm_mgr); + struct amdgpu_spm_base *spm = &spm_cntr->spm[inst]; + struct amdgpu_device *adev = prof_xcp_mgr->adev; + unsigned long flags; + u32 size_to_copy; + u32 ring_wptr; + u32 ring_rptr; + bool is_started; + int ret = 0; + + /* Snapshot is_spm_started and ring_rptr together under spm_irq_lock to prevent + * _spm_stop from resetting ring_rptr=0 between the check and access. + * If SPM is not started, ring_rptr is stale and must not be accessed. + */ + spin_lock_irqsave(&spm_mgr->spm_irq_lock, flags); + is_started = spm->is_spm_started; + ring_rptr = spm->ring_rptr; + spin_unlock_irqrestore(&spm_mgr->spm_irq_lock, flags); + + if (!is_started) + return 0; + + ring_wptr = READ_ONCE(((u32 *)spm->cpu_addr)[0]); + dma_rmb(); + + /* Validate hardware-provided wptr to prevent OOB kernel read. + * Corrupted wptr could cause size_to_copy to exceed ring buffer + * bounds, leaking kernel memory to userspace. + */ + if (ring_wptr >= spm->ring_size) { + spm->has_data_loss = true; + dev_dbg(adev->dev, "[SPM#%d] Invalid wptr:0x%x >= ring_size:0x%x\n", + inst, ring_wptr, spm->ring_size); + goto exit; + } + + /* SPM might stall if we cannot copy data out of SPM ringbuffer. + * spm->has_data_loss is only a hint here since stall is only a + * possibility and data loss might not happen. But it is a useful + * hint for user mode profiler to take extra actions. + */ + if (!spm->has_user_buf || spm->is_user_buf_filled) { + spm->has_data_loss = true; + /* Mark user buffer as filled when no user buffer was set + * or buffer is already full, signaling profiler to stop. + */ + if (!spm->is_user_buf_filled) + spm->is_user_buf_filled = true; + dev_dbg(adev->dev, "[SPM#%d] [%d|%d] rptr:0x%x--wptr:0x%x\n", inst, + spm->has_user_buf, spm->is_user_buf_filled, ring_rptr, ring_wptr); + goto exit; + } + + if (ring_rptr == ring_wptr) + goto exit; + + if (ring_wptr > ring_rptr) { + size_to_copy = ring_wptr - ring_rptr; + ret = amdgpu_spm_data_copy(spm_mgr, spm_cntr, ring_rptr, size_to_copy, inst); + } else { + /* Copy tail segment [rptr..ring_size) */ + size_to_copy = spm->ring_size - ring_rptr; + ret = amdgpu_spm_data_copy(spm_mgr, spm_cntr, ring_rptr, size_to_copy, inst); + + /* Copy wrapped segment [0..ring_wptr) if tail copy completed */ + if (spm->ring_size == READ_ONCE(spm->ring_rptr)) { + if (ring_wptr == 0) { + /* No wrapped data; reset rptr to start. */ + spin_lock_irqsave(&spm_mgr->spm_irq_lock, flags); + WRITE_ONCE(spm->ring_rptr, 0); + spin_unlock_irqrestore(&spm_mgr->spm_irq_lock, flags); + goto exit; + } + /* Reset rptr to 0 before copying wrapped segment. */ + spin_lock_irqsave(&spm_mgr->spm_irq_lock, flags); + WRITE_ONCE(spm->ring_rptr, 0); + spin_unlock_irqrestore(&spm_mgr->spm_irq_lock, flags); + + size_to_copy = ring_wptr; + if (!ret && !spm->is_user_buf_filled) + ret = amdgpu_spm_data_copy(spm_mgr, spm_cntr, 0, + size_to_copy, inst); + } + } + +exit: + /* Read rptr under spm_irq_lock to pair with the snapshot above. */ + spin_lock_irqsave(&spm_mgr->spm_irq_lock, flags); + ring_rptr = spm->ring_rptr; + spin_unlock_irqrestore(&spm_mgr->spm_irq_lock, flags); + amdgpu_rlc_spm_set_rdptr(adev, inst, ring_rptr); + return ret; +} + static void amdgpu_spm_work(struct work_struct *work) { - /* TODO: dump spm ring buffer to user buffer */ + struct amdgpu_spm_mgr *spm_mgr = container_of(work, + struct amdgpu_spm_mgr, + spm_work); + struct amdgpu_profiler_xcp_mgr *prof_xcp_mgr = + to_prof_xcp_mgr(spm_mgr, spm_mgr); + struct amdgpu_device *adev = prof_xcp_mgr->adev; + u32 xcp_id = prof_xcp_mgr->xcp_id; + struct amdgpu_spm_cntr *spm_cntr; + struct task_struct *lead_thread; + struct mm_struct *mm = NULL; + unsigned long flags; + u32 xcc_mask; + int ret; + + /* Snapshot spm_cntr and lead_thread under spm_irq_lock. + * Protects against amdgpu_spm_release() racing: release sets both + * to NULL under this lock, so either we see both non-NULL (safe to + * proceed) or one/both NULL (bail out). + */ + spin_lock_irqsave(&spm_mgr->spm_irq_lock, flags); + spm_cntr = spm_mgr->spm_cntr; + lead_thread = spm_mgr->lead_thread; + spin_unlock_irqrestore(&spm_mgr->spm_irq_lock, flags); + + if (!spm_cntr || !lead_thread) + return; + + ret = amdgpu_xcp_get_xcc_mask(adev, xcp_id, &xcc_mask); + if (ret) { + dev_err(adev->dev, "Failed to get XCC mask for XCP %u\n", xcp_id); + return; + } + + mm = get_task_mm(lead_thread); + if (mm) { + kthread_use_mm(mm); + { /* attach mm */ + bool filled; + int inst; + + mutex_lock(&spm_cntr->spm_worker_mutex); + WRITE_ONCE(spm_cntr->are_users_buf_filled, false); + /* XCC instance index space is consistent across: + * - acquire: for_each_inst(inst, xcc_mask) -> spm[inst] + * - drain work: for_each_inst(inst, xcc_mask) -> spm[inst] + * - interrupt: spm[xcc_id] where xcc_id is HW XCC instance + * All use the same XCC-instance index space from xcc_mask. + */ + for_each_inst(inst, xcc_mask) { + struct amdgpu_spm_base *spm = &spm_cntr->spm[inst]; + + /* Errors logged but not propagated: this is async work, + * copy_to_user errors are surfaced via has_data_loss flag. + */ + ret = amdgpu_spm_read_ring_buffer(spm_mgr, spm_cntr, inst); + if (ret) + dev_dbg(adev->dev, + "[SPM#%d] ring buffer drain failed: %d\n", + inst, ret); + if (spm->is_user_buf_filled) + WRITE_ONCE(spm_cntr->are_users_buf_filled, true); + } + filled = READ_ONCE(spm_cntr->are_users_buf_filled); + mutex_unlock(&spm_cntr->spm_worker_mutex); + if (filled) { + dev_dbg(adev->dev, "SPM wake up buffer work queue.\n"); + wake_up(&spm_cntr->spm_buf_wq); + } + } /* detach mm */ + kthread_unuse_mm(mm); + /* release the mm structure */ + mmput(mm); + } } int amdgpu_spm_ioctl(struct drm_device *dev, struct drm_amdgpu_spm_args *args, @@ -712,6 +932,10 @@ static int amdgpu_spm_set_dest_buf(struct amdgpu_spm_mgr *spm_mgr, void *data, mutex_unlock(&spm_cntr->spm_worker_mutex); } } + + if (xcc_mask & ~started_mask) + flush_work(&spm_mgr->spm_work); + goto out; start_failure: -- 2.34.1
