Not emitting the spm update via the ring causes CI flakiness on gfx11. Add back the SPM ring emission, but skip the MMIO path if ring is specified. Do this for all chips with gfxoff.
v2: drop ring code in gfx 12.1. We aren't supporting KCQs on 12.1 so the code is not used. v3: emit via the ring for all chips with gfxoff v4: skip MMIO writes when ring is non-NULL, fix ib emit size Cc: Vitaly Prosyak <[email protected]> Cc: Pierre-Eric Pelloux-Prayer <[email protected]> Cc: Sunil Khatri <[email protected]> Cc: Christian König <[email protected]> Fixes: 2fb5a4572c76 ("drm/amdgpu: remove the now-redundant runtime SPM VMID update") Signed-off-by: Alex Deucher <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 8 ++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 16 ++++++++++++++-- drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 17 ++++++++++++++--- drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c | 18 ++++++++++++++++-- drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c | 10 +++++++--- drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c | 7 +++++-- drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c | 8 ++++++-- drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 17 +++++++++++++++-- drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c | 7 +++++-- 10 files changed, 91 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index f98a7c6797b3c..ae028eeacaed7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -121,6 +121,10 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, struct amdgpu_job *job, struct dma_fence **f) { struct amdgpu_device *adev = ring->adev; + bool emit_spm_needed = adev->gfx.rlc.funcs && + adev->gfx.rlc.funcs->update_spm_vmid && + (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE || + ring->funcs->type == AMDGPU_RING_TYPE_GFX); struct amdgpu_ib *ib; struct dma_fence *tmp = NULL; struct amdgpu_fence *af; @@ -209,6 +213,10 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, struct amdgpu_job *job, if (need_pipe_sync) amdgpu_ring_emit_pipeline_sync(ring); + if (emit_spm_needed) + adev->gfx.rlc.funcs->update_spm_vmid(adev, ring->xcc_id, ring, + adev->vm_manager.spm_vmid); + if (emit_gds_needed) amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base, job->gds_size, job->gws_base, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h index e38a8ea02c520..2ae2c1d0f702b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h @@ -292,7 +292,7 @@ struct amdgpu_rlc_funcs { void (*reset)(struct amdgpu_device *adev); void (*start)(struct amdgpu_device *adev); void (*update_spm_vmid)(struct amdgpu_device *adev, int xcc_id, - unsigned int vmid); + struct amdgpu_ring *ring, unsigned vmid); bool (*is_rlcg_access_range)(struct amdgpu_device *adev, uint32_t reg); }; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index 7683e8a0a4380..b28dd5eaf1bbd 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -8187,8 +8187,18 @@ static void gfx_v10_0_update_spm_vmid_internal(struct amdgpu_device *adev, } static void gfx_v10_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, - unsigned int vmid) -{ + struct amdgpu_ring *ring, unsigned int vmid) +{ + if (ring) { + if ((ring->funcs->type == AMDGPU_RING_TYPE_GFX) || + (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)) { + u32 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL); + u32 data = (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; + amdgpu_ring_emit_wreg(ring, reg, data); + } + return; + } + amdgpu_gfx_off_ctrl(adev, false); gfx_v10_0_update_spm_vmid_internal(adev, vmid); @@ -9692,6 +9702,7 @@ static const struct amdgpu_ring_funcs gfx_v10_0_ring_funcs_gfx = { .get_wptr = gfx_v10_0_ring_get_wptr_gfx, .set_wptr = gfx_v10_0_ring_set_wptr_gfx, .emit_frame_size = /* totally 242 maximum if 16 IBs */ + 5 + /* update_spm_vmid */ 5 + /* COND_EXEC */ 7 + /* PIPELINE_SYNC */ SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + @@ -9749,6 +9760,7 @@ static const struct amdgpu_ring_funcs gfx_v10_0_ring_funcs_compute = { .get_wptr = gfx_v10_0_ring_get_wptr_compute, .set_wptr = gfx_v10_0_ring_set_wptr_compute, .emit_frame_size = + 5 + /* update_spm_vmid */ 20 + /* gfx_v10_0_ring_emit_gds_switch */ 7 + /* gfx_v10_0_ring_emit_hdp_flush */ 5 + /* hdp invalidate */ diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index 5d068a9d9c72c..c0e74e12cd15d 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -363,6 +363,7 @@ static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring, static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id); static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id); static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, + struct amdgpu_ring *ring, unsigned int vmid); static void gfx_v11_0_userq_priv_fault_work(struct work_struct *work); static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev, @@ -2606,7 +2607,7 @@ static int gfx_v11_0_rlc_resume(struct amdgpu_device *adev) * The RLC comes up with RLC_SPM_MC_CNTL cleared, program the SPM VMID * once it runs, also after a GPU reset or a resume. */ - gfx_v11_0_update_spm_vmid(adev, 0, adev->vm_manager.spm_vmid); + gfx_v11_0_update_spm_vmid(adev, 0, NULL, adev->vm_manager.spm_vmid); return 0; } @@ -5615,12 +5616,22 @@ static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev, } static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, - unsigned int vmid) + struct amdgpu_ring *ring, unsigned vmid) { u32 reg, pre_data, data; - amdgpu_gfx_off_ctrl(adev, false); reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL); + + if (ring) { + if ((ring->funcs->type == AMDGPU_RING_TYPE_GFX) || + (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)) { + data = (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; + amdgpu_ring_emit_wreg(ring, reg, data); + } + return; + } + + amdgpu_gfx_off_ctrl(adev, false); if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) pre_data = RREG32_NO_KIQ(reg); else diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c index df2fb09bbcd86..0c212b9a56398 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c @@ -287,6 +287,7 @@ static void gfx_v12_0_ring_invalidate_tlbs(struct amdgpu_ring *ring, static void gfx_v12_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id); static void gfx_v12_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id); static void gfx_v12_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, + struct amdgpu_ring *ring, unsigned int vmid); static void gfx_v12_0_userq_priv_fault_work(struct work_struct *work); static void gfx_v12_0_update_perf_clk(struct amdgpu_device *adev, @@ -2158,7 +2159,7 @@ static int gfx_v12_0_rlc_resume(struct amdgpu_device *adev) * The RLC comes up with RLC_SPM_MC_CNTL cleared, program the SPM VMID * once it runs, also after a GPU reset or a resume. */ - gfx_v12_0_update_spm_vmid(adev, 0, adev->vm_manager.spm_vmid); + gfx_v12_0_update_spm_vmid(adev, 0, NULL, adev->vm_manager.spm_vmid); return 0; } @@ -3978,12 +3979,23 @@ static void gfx_v12_0_update_perf_clk(struct amdgpu_device *adev, static void gfx_v12_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, + struct amdgpu_ring *ring, unsigned vmid) { u32 reg, data; - amdgpu_gfx_off_ctrl(adev, false); reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL); + + if (ring) { + if ((ring->funcs->type == AMDGPU_RING_TYPE_GFX) || + (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)) { + data = (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; + amdgpu_ring_emit_wreg(ring, reg, data); + } + return; + } + + amdgpu_gfx_off_ctrl(adev, false); if (amdgpu_sriov_is_pp_one_vf(adev)) data = RREG32_NO_KIQ(reg); else @@ -5322,6 +5334,7 @@ static const struct amdgpu_ring_funcs gfx_v12_0_ring_funcs_gfx = { .get_wptr = gfx_v12_0_ring_get_wptr_gfx, .set_wptr = gfx_v12_0_ring_set_wptr_gfx, .emit_frame_size = /* totally 242 maximum if 16 IBs */ + 5 + /* update_spm_vmid */ 5 + /* COND_EXEC */ 7 + /* PIPELINE_SYNC */ SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + @@ -5369,6 +5382,7 @@ static const struct amdgpu_ring_funcs gfx_v12_0_ring_funcs_compute = { .get_wptr = gfx_v12_0_ring_get_wptr_compute, .set_wptr = gfx_v12_0_ring_set_wptr_compute, .emit_frame_size = + 5 + /* update_spm_vmid */ 7 + /* gfx_v12_0_ring_emit_hdp_flush */ 5 + /* hdp invalidate */ 7 + /* gfx_v12_0_ring_emit_pipeline_sync */ diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c index 1f57d0fa2c044..227942e87cc8f 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c @@ -217,7 +217,7 @@ static int gfx_v12_1_init_cp_compute_microcode_bo(struct amdgpu_device *adev); static void gfx_v12_1_xcc_update_medium_grain_clock_gating( struct amdgpu_device *adev, bool enable, int xcc_id, bool force); static void gfx_v12_1_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, - unsigned int vmid); + struct amdgpu_ring *ring, unsigned vmid); static void gfx_v12_1_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, int mem_space, int opt, uint32_t addr0, @@ -1866,7 +1866,7 @@ static int gfx_v12_1_xcc_rlc_resume(struct amdgpu_device *adev, int r; /* init spm vmid with 0xf */ - gfx_v12_1_update_spm_vmid(adev, xcc_id, adev->vm_manager.spm_vmid); + gfx_v12_1_update_spm_vmid(adev, xcc_id, NULL, adev->vm_manager.spm_vmid); if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { gfx_v12_1_xcc_init_csb(adev, xcc_id); @@ -2945,10 +2945,14 @@ static void gfx_v12_1_update_perf_clk(struct amdgpu_device *adev, static void gfx_v12_1_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, - unsigned int vmid) + struct amdgpu_ring *ring, + unsigned vmid) { u32 reg, data; + if (ring) + return; + reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL); if (amdgpu_sriov_is_pp_one_vf(adev)) data = RREG32_NO_KIQ(reg); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c index 58135757190c8..3c9e478d5f6e7 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c @@ -3477,16 +3477,19 @@ static int gfx_v7_0_rlc_resume(struct amdgpu_device *adev) /* init spm vmid with 0xf */ if (adev->gfx.rlc.funcs->update_spm_vmid) - adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, adev->vm_manager.spm_vmid); + adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, adev->vm_manager.spm_vmid); return 0; } static void gfx_v7_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, - unsigned int vmid) + struct amdgpu_ring *ring, unsigned vmid) { u32 data; + if (ring) + return; + amdgpu_gfx_off_ctrl(adev, false); data = RREG32(mmRLC_SPM_VMID); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index df935827b002a..e818280139791 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -4084,7 +4084,8 @@ static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev) /* init spm vmid with 0xf */ if (adev->gfx.rlc.funcs->update_spm_vmid) - adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, adev->vm_manager.spm_vmid); + adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, adev->vm_manager.spm_vmid); + return 0; } @@ -5483,10 +5484,13 @@ static void gfx_v8_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id) } static void gfx_v8_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, - unsigned int vmid) + struct amdgpu_ring *ring, unsigned vmid) { u32 data; + if (ring) + return; + amdgpu_gfx_off_ctrl(adev, false); if (amdgpu_sriov_is_pp_one_vf(adev)) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 2e2673a6dbbf5..be45c10b6f579 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -5227,8 +5227,18 @@ static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev, } static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, - unsigned int vmid) -{ + struct amdgpu_ring *ring, unsigned int vmid) +{ + if (ring) { + if ((ring->funcs->type == AMDGPU_RING_TYPE_GFX) || + (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)) { + u32 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL); + u32 data = (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; + amdgpu_ring_emit_wreg(ring, reg, data); + } + return; + } + amdgpu_gfx_off_ctrl(adev, false); gfx_v9_0_update_spm_vmid_internal(adev, vmid); @@ -7548,6 +7558,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { .get_wptr = gfx_v9_0_ring_get_wptr_gfx, .set_wptr = gfx_v9_0_ring_set_wptr_gfx, .emit_frame_size = /* totally 242 maximum if 16 IBs */ + 5 + /* update_spm_vmid */ 5 + /* COND_EXEC */ 13 + /* PIPELINE_SYNC */ SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + @@ -7604,6 +7615,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_sw_ring_funcs_gfx = { .get_wptr = amdgpu_sw_ring_get_wptr_gfx, .set_wptr = amdgpu_sw_ring_set_wptr_gfx, .emit_frame_size = /* totally 242 maximum if 16 IBs */ + 5 + /* update_spm_vmid */ 5 + /* COND_EXEC */ 13 + /* PIPELINE_SYNC */ SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + @@ -7663,6 +7675,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { .get_wptr = gfx_v9_0_ring_get_wptr_compute, .set_wptr = gfx_v9_0_ring_set_wptr_compute, .emit_frame_size = + 5 + /* update_spm_vmid */ 20 + /* gfx_v9_0_ring_emit_gds_switch */ 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 5 + /* hdp invalidate */ diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c index 151336e6094f7..f28ffe8a5ac55 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c @@ -1586,7 +1586,7 @@ static int gfx_v9_4_3_xcc_rlc_resume(struct amdgpu_device *adev, int xcc_id) /* init spm vmid with 0xf */ if (adev->gfx.rlc.funcs->update_spm_vmid) - adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, adev->vm_manager.spm_vmid); + adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, adev->vm_manager.spm_vmid); return 0; } @@ -1609,10 +1609,13 @@ static int gfx_v9_4_3_rlc_resume(struct amdgpu_device *adev) } static void gfx_v9_4_3_update_spm_vmid(struct amdgpu_device *adev, - int inst, unsigned int vmid) + int inst, struct amdgpu_ring *ring, unsigned int vmid) { u32 reg, pre_data, data; + if (ring) + return; + reg = SOC15_REG_OFFSET(GC, GET_INST(GC, inst), regRLC_SPM_MC_CNTL); if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) pre_data = RREG32_NO_KIQ(reg); -- 2.56.1
