Every GFX IP programs the RLC SPM (Streaming Performance Monitor) block through its own registers, so without a common layer the SPM manager would have to open-code that per-IP sequencing at each call site. Add a hardware-agnostic layer: the manager drives SPM through a few generic entry points and each IP only supplies register programming via an amdgpu_spm_funcs table in adev->gfx.spmfuncs.
All SPM programming is issued through the KIQ ring so it is serialised against other KIQ users and the RLC firmware; each op emits a polling fence and waits so the firmware has consumed the packet before the call returns. The per-IP callbacks run under the KIQ ring_lock and may only emit packets, and set_spm_config_size tells the generic layer how much ring space one op needs. acquire reserves a VMID on the GFX hub for the SPM stream; release stops the stream and waits for that stop to complete before freeing the VMID, so the RLC can never keep streaming into a VMID that has been reused. Signed-off-by: James Zhu <[email protected]> Reviewed-by: Bing Ma <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 15 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c | 230 ++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h | 7 + 3 files changed, 252 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h index aefd4f03b443..fd87075962f9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h @@ -174,6 +174,20 @@ struct amdgpu_kiq { void *mqd_backup; }; +struct amdgpu_spm_funcs { + void (*start)(struct amdgpu_device *adev, int xcc_id); + void (*stop)(struct amdgpu_device *adev, int xcc_id); + void (*set_rdptr)(struct amdgpu_device *adev, int xcc_id, u32 rptr); + void (*set_spm_perfmon_ring_buf)(struct amdgpu_device *adev, int xcc_id, + u64 gpu_addr, u32 size); + /* Dwords amdgpu_rlc_spm_kiq_submit() reserves on the KIQ ring for one + * submission. Must cover the largest of the callbacks above plus the + * polling fence emitted after them; amdgpu_ring_write() has no bounds + * check, so a short value silently corrupts the ring. + */ + u32 set_spm_config_size; +}; + /* * GFX configurations */ @@ -429,6 +443,7 @@ struct amdgpu_gfx { struct amdgpu_mec_bitmap mec_bitmap[AMDGPU_MAX_GC_INSTANCES]; struct amdgpu_kiq kiq[AMDGPU_MAX_GC_INSTANCES]; struct amdgpu_imu imu; + const struct amdgpu_spm_funcs *spmfuncs; bool rs64_enable; /* firmware format */ const struct firmware *me_fw; /* ME firmware */ uint32_t me_fw_version; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c index 002fae3c380e..79641e537fbb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c @@ -622,3 +622,233 @@ void amdgpu_init_rlc_reg_funcs(struct amdgpu_device *adev) else adev->gfx.rlc.reg_funcs = &amdgpu_rlc_reg_funcs; } + +/** + * amdgpu_rlc_spm_kiq_submit - helper for SPM KIQ command submission + * @adev: amdgpu device pointer + * @xcc_id: XCC instance ID + * @emit_fn: callback to emit commands into the ring + * @emit_ctx: context pointer passed to emit_fn + * + * Common pattern for SPM KIQ operations: lock, alloc, emit, fence, commit, + * unlock, wait. Centralizes error/undo paths. + * + * Callers guarantee spmfuncs is non-NULL. + * + * Return: 0 on success, negative error code on failure. + */ +static int amdgpu_rlc_spm_kiq_submit(struct amdgpu_device *adev, int xcc_id, + void (*emit_fn)(struct amdgpu_device *, int, void *), + void *emit_ctx) +{ + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[xcc_id].ring; + unsigned long flags; + signed long r; + u32 seq; + int ret; + + spin_lock_irqsave(&adev->gfx.kiq[xcc_id].ring_lock, flags); + ret = amdgpu_ring_alloc(kiq_ring, adev->gfx.spmfuncs->set_spm_config_size); + if (ret) { + spin_unlock_irqrestore(&adev->gfx.kiq[xcc_id].ring_lock, flags); + return ret; + } + + emit_fn(adev, xcc_id, emit_ctx); + + ret = amdgpu_fence_emit_polling(kiq_ring, &seq, MAX_KIQ_REG_WAIT); + if (ret) { + amdgpu_ring_undo(kiq_ring); + spin_unlock_irqrestore(&adev->gfx.kiq[xcc_id].ring_lock, flags); + dev_warn(adev->dev, "%s SPM KIQ fence emit failed (%d)\n", __func__, ret); + return ret; + } + + amdgpu_ring_commit(kiq_ring); + spin_unlock_irqrestore(&adev->gfx.kiq[xcc_id].ring_lock, flags); + + r = amdgpu_fence_wait_polling(kiq_ring, seq, MAX_KIQ_REG_WAIT); + if (r < 1) { + dev_warn(adev->dev, "%s SPM KIQ fence timeout (seq=%u)\n", __func__, seq); + return -ETIMEDOUT; + } + + return 0; +} + +static void amdgpu_rlc_spm_emit_start(struct amdgpu_device *adev, int xcc_id, + void *ctx) +{ + adev->gfx.spmfuncs->start(adev, xcc_id); +} + +static void amdgpu_rlc_spm_emit_stop(struct amdgpu_device *adev, int xcc_id, + void *ctx) +{ + adev->gfx.spmfuncs->stop(adev, xcc_id); +} + +/** + * amdgpu_rlc_spm_cntl - start or stop SPM hardware stream + * @adev: amdgpu device pointer + * @xcc_id: XCC instance ID (must be < adev->gfx.xcc_mask bit count) + * @cntl: true to start, false to stop + * + * Caller must ensure xcc_id is valid - no bounds checking is performed. + * + * Return: 0 on success, negative error code on failure. + */ +int amdgpu_rlc_spm_cntl(struct amdgpu_device *adev, int xcc_id, bool cntl) +{ + if (!adev->gfx.spmfuncs) + return -EINVAL; + + if (!(cntl ? adev->gfx.spmfuncs->start : adev->gfx.spmfuncs->stop)) + return -EINVAL; + + if (!adev->gfx.spmfuncs->set_spm_config_size) { + if (cntl) + adev->gfx.spmfuncs->start(adev, xcc_id); + else + adev->gfx.spmfuncs->stop(adev, xcc_id); + return 0; + } + + return amdgpu_rlc_spm_kiq_submit(adev, xcc_id, + cntl ? amdgpu_rlc_spm_emit_start : amdgpu_rlc_spm_emit_stop, NULL); +} + +struct amdgpu_rlc_spm_rdptr_ctx { + u32 rptr; +}; + +static void amdgpu_rlc_spm_emit_set_rdptr(struct amdgpu_device *adev, int xcc_id, void *ctx) +{ + struct amdgpu_rlc_spm_rdptr_ctx *rdptr_ctx = ctx; + + adev->gfx.spmfuncs->set_rdptr(adev, xcc_id, rdptr_ctx->rptr); +} + +/** + * amdgpu_rlc_spm_set_rdptr - update SPM ring read pointer + * @adev: amdgpu device pointer + * @xcc_id: XCC instance ID (must be < adev->gfx.xcc_mask bit count) + * @rptr: new read pointer offset + * + * Caller must ensure xcc_id is valid - no bounds checking is performed. + */ +void amdgpu_rlc_spm_set_rdptr(struct amdgpu_device *adev, int xcc_id, u32 rptr) +{ + struct amdgpu_rlc_spm_rdptr_ctx rdptr_ctx = { .rptr = rptr }; + int ret; + + if (!adev->gfx.spmfuncs || !adev->gfx.spmfuncs->set_rdptr) + return; + + if (!adev->gfx.spmfuncs->set_spm_config_size) { + adev->gfx.spmfuncs->set_rdptr(adev, xcc_id, rptr); + return; + } + + ret = amdgpu_rlc_spm_kiq_submit(adev, xcc_id, amdgpu_rlc_spm_emit_set_rdptr, &rdptr_ctx); + if (ret) + dev_warn(adev->dev, "Failed to update SPM rdptr to 0x%x on XCC %d (%d)\n", + rptr, xcc_id, ret); +} + +struct amdgpu_rlc_spm_perfmon_ring_ctx { + u64 gpu_addr; + u32 size; +}; + +static void amdgpu_rlc_spm_emit_perfmon_ring_buf(struct amdgpu_device *adev, int xcc_id, void *ctx) +{ + struct amdgpu_rlc_spm_perfmon_ring_ctx *ring_ctx = ctx; + + adev->gfx.spmfuncs->set_spm_perfmon_ring_buf(adev, xcc_id, + ring_ctx->gpu_addr, + ring_ctx->size); +} + +/** + * amdgpu_rlc_spm_acquire - set up SPM for a specific XCC instance + * @adev: amdgpu device pointer + * @xcc_id: XCC instance ID (must be < adev->gfx.xcc_mask bit count) + * @vm: virtual memory context for VMID allocation + * @gpu_addr: GPU address of SPM ring buffer + * @size: size of SPM ring buffer in bytes + * + * Allocates a reserved VMID, initializes SPM VMID to 0, and programs the + * ring buffer address and size. On failure, reverts VMID to 0xf and frees + * the reserved VMID. + * + * Caller must ensure xcc_id is valid - no bounds checking is performed. + * + * Return: 0 on success, negative error code on failure. + */ +int amdgpu_rlc_spm_acquire(struct amdgpu_device *adev, int xcc_id, + struct amdgpu_vm *vm, u64 gpu_addr, u32 size) +{ + struct amdgpu_rlc_spm_perfmon_ring_ctx ring_ctx = { .gpu_addr = gpu_addr, .size = size }; + int r; + + if (!adev->gfx.spmfuncs || + !adev->gfx.spmfuncs->set_spm_perfmon_ring_buf || + !adev->gfx.rlc.funcs || + !adev->gfx.rlc.funcs->update_spm_vmid) + return -EINVAL; + + r = amdgpu_vmid_alloc_reserved(adev, vm, AMDGPU_GFXHUB(xcc_id)); + if (r) + return r; + + /* init spm vmid with 0x0 (outside ring_lock to avoid sleep-in-atomic) */ + adev->gfx.rlc.funcs->update_spm_vmid(adev, xcc_id, NULL, 0); + + if (!adev->gfx.spmfuncs->set_spm_config_size) { + adev->gfx.spmfuncs->set_spm_perfmon_ring_buf(adev, xcc_id, gpu_addr, size); + return 0; + } + + /* set spm ring registers */ + r = amdgpu_rlc_spm_kiq_submit(adev, xcc_id, amdgpu_rlc_spm_emit_perfmon_ring_buf, &ring_ctx); + if (r) + goto err_revert; + + return 0; + +err_revert: + adev->gfx.rlc.funcs->update_spm_vmid(adev, xcc_id, NULL, 0xf); + amdgpu_vmid_free_reserved(adev, vm, AMDGPU_GFXHUB(xcc_id)); + return r; +} + +/** + * amdgpu_rlc_spm_release - tear down SPM for a specific XCC instance + * @adev: amdgpu device pointer + * @xcc_id: XCC instance ID (must be < adev->gfx.xcc_mask bit count) + * @vm: virtual memory context for VMID release + * + * Stops the SPM stream via KIQ, waits for completion, then reverts SPM VMID + * to 0xf and frees the reserved VMID. Waiting ensures the firmware has stopped + * SPM before the VMID is reclaimed, preventing VMID reuse while hardware is + * still active. On stop failure, still reverts VMID and frees reservation + * (best-effort cleanup) to prevent resource leaks. + * + * Caller must ensure xcc_id is valid - no bounds checking is performed. + */ +void amdgpu_rlc_spm_release(struct amdgpu_device *adev, int xcc_id, struct amdgpu_vm *vm) +{ + int r; + + r = amdgpu_rlc_spm_cntl(adev, xcc_id, false); + if (r) + dev_warn(adev->dev, "SPM stop failed on XCC %d (error %d), proceeding with VMID cleanup\n", + xcc_id, r); + + /* revert spm vmid with 0xf */ + if (adev->gfx.rlc.funcs && adev->gfx.rlc.funcs->update_spm_vmid) + adev->gfx.rlc.funcs->update_spm_vmid(adev, xcc_id, NULL, 0xf); + + amdgpu_vmid_free_reserved(adev, vm, AMDGPU_GFXHUB(xcc_id)); +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h index 959d60c90dcd..18b925a0d164 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h @@ -26,6 +26,8 @@ #include "clearstate_defs.h" +struct amdgpu_vm; + #define AMDGPU_MAX_RLC_INSTANCES 8 /* firmware ID used in rlc toc */ @@ -384,4 +386,9 @@ int amdgpu_gfx_rlc_init_microcode(struct amdgpu_device *adev, void amdgpu_early_init_rlc_reg_funcs(struct amdgpu_device *adev); void amdgpu_init_rlc_reg_funcs(struct amdgpu_device *adev); +int amdgpu_rlc_spm_cntl(struct amdgpu_device *adev, int xcc_id, bool cntl); +int amdgpu_rlc_spm_acquire(struct amdgpu_device *adev, int xcc_id, + struct amdgpu_vm *vm, u64 gpu_addr, u32 size); +void amdgpu_rlc_spm_release(struct amdgpu_device *adev, int xcc_id, struct amdgpu_vm *vm); +void amdgpu_rlc_spm_set_rdptr(struct amdgpu_device *adev, int xcc_id, u32 rptr); #endif -- 2.34.1
