priv_reg / priv_inst / bad_op and (on v11+) userq EOP IRQs are acquired in late_init but released in hw_fini. This split forced gfx_v9_0_hw_fini() to defensively guard each put with amdgpu_irq_enabled() because hw_fini runs on paths that may not reach late_init.
amdgpu_ip_block_hw_fini() only runs after hw_init returns success, and suspend / resume cycle the refs through the same path, so hw_init / hw_fini pair without any extra tracking. Move the gets there and drop the guards. While here, fix the pre-existing partial-failure leak in set_userq_eop_interrupts() (gfx11 / 12_0 / 12_1). amdgpu_irq_get() increments the refcount before calling .set, so a failure partway through the loop leaves earlier successful gets stranded. Track the loop position and roll back on the enable path. Signed-off-by: Yunxiang Li <[email protected]> --- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 43 +++---- drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 162 ++++++++++++++---------- drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c | 162 ++++++++++++++---------- drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c | 114 +++++++++-------- drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 39 +++--- drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c | 35 ++--- 6 files changed, 315 insertions(+), 240 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index 58c69dcb527f7..0780c5e5de4ff 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -7523,32 +7523,50 @@ static int gfx_v10_0_hw_init(struct amdgpu_ip_block *ip_block) r = gfx_v10_0_cp_resume(adev); if (r) return r; if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(10, 3, 0)) gfx_v10_3_program_pbb_mode(adev); if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 3, 0) && !amdgpu_sriov_vf(adev)) gfx_v10_3_set_power_brake_sequence(adev); + r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); + if (r) + return r; + + r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); + if (r) + goto err_priv_inst; + + r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); + if (r) + goto err_bad_op; + + return 0; + +err_bad_op: + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); +err_priv_inst: + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); return r; } static int gfx_v10_0_hw_fini(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; cancel_delayed_work_sync(&adev->gfx.idle_work); - amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); - amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); /* WA added for Vangogh asic fixing the SMU suspend failure * It needs to set power gating again during gfxoff control * otherwise the gfxoff disallowing will be failed to set. */ if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(10, 3, 1)) gfx_v10_0_set_powergating_state(ip_block, AMD_PG_STATE_UNGATE); if (!adev->no_hw_access) { if (amdgpu_async_gfx_ring) { @@ -7830,40 +7848,20 @@ static int gfx_v10_0_early_init(struct amdgpu_ip_block *ip_block) gfx_v10_0_set_gds_init(adev); gfx_v10_0_set_rlc_funcs(adev); gfx_v10_0_set_mqd_funcs(adev); /* init rlcg reg access ctrl */ gfx_v10_0_init_rlcg_reg_access_ctrl(adev); return gfx_v10_0_init_microcode(adev); } -static int gfx_v10_0_late_init(struct amdgpu_ip_block *ip_block) -{ - struct amdgpu_device *adev = ip_block->adev; - int r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); - if (r) - return r; - - return 0; -} - static bool gfx_v10_0_is_rlc_enabled(struct amdgpu_device *adev) { uint32_t rlc_cntl; /* if RLC is not enabled, do nothing */ rlc_cntl = RREG32_SOC15(GC, 0, mmRLC_CNTL); return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false; } static void gfx_v10_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id) @@ -9798,21 +9796,20 @@ static void gfx_v10_0_ring_begin_use(struct amdgpu_ring *ring) static void gfx_v10_0_ring_end_use(struct amdgpu_ring *ring) { amdgpu_gfx_profile_ring_end_use(ring); amdgpu_gfx_enforce_isolation_ring_end_use(ring); } static const struct amd_ip_funcs gfx_v10_0_ip_funcs = { .name = "gfx_v10_0", .early_init = gfx_v10_0_early_init, - .late_init = gfx_v10_0_late_init, .sw_init = gfx_v10_0_sw_init, .sw_fini = gfx_v10_0_sw_fini, .hw_init = gfx_v10_0_hw_init, .hw_fini = gfx_v10_0_hw_fini, .suspend = gfx_v10_0_suspend, .resume = gfx_v10_0_resume, .is_idle = gfx_v10_0_is_idle, .wait_for_idle = gfx_v10_0_wait_for_idle, .soft_reset = gfx_v10_0_soft_reset, .set_clockgating_state = gfx_v10_0_set_clockgating_state, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index fabdbbd0abb7c..de0bf6e1f64d8 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -4800,20 +4800,92 @@ static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev) data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG); data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK; WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data); data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG); data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK; WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data); } +static int gfx_v11_0_set_userq_eop_interrupts(struct amdgpu_device *adev, + bool enable) +{ + unsigned int irq_type; + int m, p, r; + + if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) { + for (m = 0; m < adev->gfx.me.num_me; m++) { + for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) { + irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; + if (enable) + r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); + else + r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + if (r) { + if (!enable) + return r; + goto err_gfx; + } + } + } + } + + if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) { + for (m = 0; m < adev->gfx.mec.num_mec; ++m) { + for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + + p; + if (enable) + r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); + else + r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + if (r) { + if (!enable) + return r; + goto err_compute; + } + } + } + } + + return 0; + +err_compute: + for (p--; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + for (m--; m >= 0; m--) { + for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + } + m = adev->gfx.me.num_me; +err_gfx: + for (p--; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + for (m--; m >= 0; m--) { + for (p = adev->gfx.me.num_pipe_per_me - 1; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + } + return r; +} + static int gfx_v11_0_hw_init(struct amdgpu_ip_block *ip_block) { int r; struct amdgpu_device *adev = ip_block->adev; amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size, adev->gfx.cleaner_shader_ptr); if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { if (adev->gfx.imu.funcs) { @@ -4897,76 +4969,57 @@ static int gfx_v11_0_hw_init(struct amdgpu_ip_block *ip_block) gfx_v11_0_tcp_harvest(adev); r = gfx_v11_0_cp_resume(adev); if (r) return r; /* get IMU version from HW if it's not set */ if (!adev->gfx.imu_fw_version) adev->gfx.imu_fw_version = RREG32_SOC15(GC, 0, regGFX_IMU_SCRATCH_0); - return r; -} - -static int gfx_v11_0_set_userq_eop_interrupts(struct amdgpu_device *adev, - bool enable) -{ - unsigned int irq_type; - int m, p, r; - - if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) { - for (m = 0; m < adev->gfx.me.num_me; m++) { - for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) { - irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; - if (enable) - r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, - irq_type); - else - r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, - irq_type); - if (r) - return r; - } - } - } - - if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) { - for (m = 0; m < adev->gfx.mec.num_mec; ++m) { - for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { - irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP - + (m * adev->gfx.mec.num_pipe_per_mec) - + p; - if (enable) - r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, - irq_type); - else - r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, - irq_type); - if (r) - return r; - } - } - } + r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); + if (r) + return r; + + r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); + if (r) + goto err_priv_inst; + + r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); + if (r) + goto err_bad_op; + + r = gfx_v11_0_set_userq_eop_interrupts(adev, true); + if (r) + goto err_userq_eop; return 0; + +err_userq_eop: + amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); +err_bad_op: + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); +err_priv_inst: + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); + return r; } static int gfx_v11_0_hw_fini(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; cancel_delayed_work_sync(&adev->gfx.idle_work); - amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); - amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); - amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); gfx_v11_0_set_userq_eop_interrupts(adev, false); + amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); if (!adev->no_hw_access) { if (amdgpu_async_gfx_ring && !adev->gfx.disable_kq) { if (amdgpu_gfx_disable_kgq(adev, 0)) DRM_ERROR("KGQ disable failed\n"); } if (amdgpu_gfx_disable_kcq(adev, 0)) DRM_ERROR("KCQ disable failed\n"); @@ -5342,44 +5395,20 @@ static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block) gfx_v11_0_set_gds_init(adev); gfx_v11_0_set_rlc_funcs(adev); gfx_v11_0_set_mqd_funcs(adev); gfx_v11_0_set_imu_funcs(adev); gfx_v11_0_init_rlcg_reg_access_ctrl(adev); return gfx_v11_0_init_microcode(adev); } -static int gfx_v11_0_late_init(struct amdgpu_ip_block *ip_block) -{ - struct amdgpu_device *adev = ip_block->adev; - int r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); - if (r) - return r; - - r = gfx_v11_0_set_userq_eop_interrupts(adev, true); - if (r) - return r; - - return 0; -} - static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev) { uint32_t rlc_cntl; /* if RLC is not enabled, do nothing */ rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL); return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false; } static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id) @@ -7194,21 +7223,20 @@ static void gfx_v11_0_ring_begin_use(struct amdgpu_ring *ring) static void gfx_v11_0_ring_end_use(struct amdgpu_ring *ring) { amdgpu_gfx_profile_ring_end_use(ring); amdgpu_gfx_enforce_isolation_ring_end_use(ring); } static const struct amd_ip_funcs gfx_v11_0_ip_funcs = { .name = "gfx_v11_0", .early_init = gfx_v11_0_early_init, - .late_init = gfx_v11_0_late_init, .sw_init = gfx_v11_0_sw_init, .sw_fini = gfx_v11_0_sw_fini, .hw_init = gfx_v11_0_hw_init, .hw_fini = gfx_v11_0_hw_fini, .suspend = gfx_v11_0_suspend, .resume = gfx_v11_0_resume, .is_idle = gfx_v11_0_is_idle, .wait_for_idle = gfx_v11_0_wait_for_idle, .soft_reset = gfx_v11_0_soft_reset, .check_soft_reset = gfx_v11_0_check_soft_reset, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c index f47928dcd8480..f66293fc675e7 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c @@ -3648,20 +3648,92 @@ static void gfx_v12_0_init_golden_registers(struct amdgpu_device *adev) if (adev->rev_id == 0) soc15_program_register_sequence(adev, golden_settings_gc_12_0_rev0, (const u32)ARRAY_SIZE(golden_settings_gc_12_0_rev0)); break; default: break; } } +static int gfx_v12_0_set_userq_eop_interrupts(struct amdgpu_device *adev, + bool enable) +{ + unsigned int irq_type; + int m, p, r; + + if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) { + for (m = 0; m < adev->gfx.me.num_me; m++) { + for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) { + irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; + if (enable) + r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); + else + r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + if (r) { + if (!enable) + return r; + goto err_gfx; + } + } + } + } + + if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) { + for (m = 0; m < adev->gfx.mec.num_mec; ++m) { + for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + + p; + if (enable) + r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); + else + r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + if (r) { + if (!enable) + return r; + goto err_compute; + } + } + } + } + + return 0; + +err_compute: + for (p--; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + for (m--; m >= 0; m--) { + for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + } + m = adev->gfx.me.num_me; +err_gfx: + for (p--; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + for (m--; m >= 0; m--) { + for (p = adev->gfx.me.num_pipe_per_me - 1; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + } + return r; +} + static int gfx_v12_0_hw_init(struct amdgpu_ip_block *ip_block) { int r; struct amdgpu_device *adev = ip_block->adev; if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) { /* RLC autoload sequence 1: Program rlc ram */ if (adev->gfx.imu.funcs->program_rlc_ram) adev->gfx.imu.funcs->program_rlc_ram(adev); @@ -3735,77 +3807,58 @@ static int gfx_v12_0_hw_init(struct amdgpu_ip_block *ip_block) /* * init golden registers and rlc resume may override some registers, * reconfig them here */ gfx_v12_0_tcp_harvest(adev); r = gfx_v12_0_cp_resume(adev); if (r) return r; - return r; -} - -static int gfx_v12_0_set_userq_eop_interrupts(struct amdgpu_device *adev, - bool enable) -{ - unsigned int irq_type; - int m, p, r; - - if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) { - for (m = 0; m < adev->gfx.me.num_me; m++) { - for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) { - irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; - if (enable) - r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, - irq_type); - else - r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, - irq_type); - if (r) - return r; - } - } - } - - if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) { - for (m = 0; m < adev->gfx.mec.num_mec; ++m) { - for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { - irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP - + (m * adev->gfx.mec.num_pipe_per_mec) - + p; - if (enable) - r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, - irq_type); - else - r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, - irq_type); - if (r) - return r; - } - } - } + r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); + if (r) + return r; + + r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); + if (r) + goto err_priv_inst; + + r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); + if (r) + goto err_bad_op; + + r = gfx_v12_0_set_userq_eop_interrupts(adev, true); + if (r) + goto err_userq_eop; return 0; + +err_userq_eop: + amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); +err_bad_op: + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); +err_priv_inst: + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); + return r; } static int gfx_v12_0_hw_fini(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; uint32_t tmp; cancel_delayed_work_sync(&adev->gfx.idle_work); - amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); - amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); - amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); gfx_v12_0_set_userq_eop_interrupts(adev, false); + amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); if (!adev->no_hw_access) { if (amdgpu_async_gfx_ring) { if (amdgpu_gfx_disable_kgq(adev, 0)) DRM_ERROR("KGQ disable failed\n"); } if (amdgpu_gfx_disable_kcq(adev, 0)) DRM_ERROR("KCQ disable failed\n"); @@ -3920,44 +3973,20 @@ static int gfx_v12_0_early_init(struct amdgpu_ip_block *ip_block) gfx_v12_0_set_irq_funcs(adev); gfx_v12_0_set_rlc_funcs(adev); gfx_v12_0_set_mqd_funcs(adev); gfx_v12_0_set_imu_funcs(adev); gfx_v12_0_init_rlcg_reg_access_ctrl(adev); return gfx_v12_0_init_microcode(adev); } -static int gfx_v12_0_late_init(struct amdgpu_ip_block *ip_block) -{ - struct amdgpu_device *adev = ip_block->adev; - int r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); - if (r) - return r; - - r = gfx_v12_0_set_userq_eop_interrupts(adev, true); - if (r) - return r; - - return 0; -} - static bool gfx_v12_0_is_rlc_enabled(struct amdgpu_device *adev) { uint32_t rlc_cntl; /* if RLC is not enabled, do nothing */ rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL); return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false; } static void gfx_v12_0_set_safe_mode(struct amdgpu_device *adev, @@ -5433,21 +5462,20 @@ static void gfx_v12_0_ring_begin_use(struct amdgpu_ring *ring) static void gfx_v12_0_ring_end_use(struct amdgpu_ring *ring) { amdgpu_gfx_profile_ring_end_use(ring); amdgpu_gfx_enforce_isolation_ring_end_use(ring); } static const struct amd_ip_funcs gfx_v12_0_ip_funcs = { .name = "gfx_v12_0", .early_init = gfx_v12_0_early_init, - .late_init = gfx_v12_0_late_init, .sw_init = gfx_v12_0_sw_init, .sw_fini = gfx_v12_0_sw_fini, .hw_init = gfx_v12_0_hw_init, .hw_fini = gfx_v12_0_hw_fini, .suspend = gfx_v12_0_suspend, .resume = gfx_v12_0_resume, .is_idle = gfx_v12_0_is_idle, .wait_for_idle = gfx_v12_0_wait_for_idle, .set_clockgating_state = gfx_v12_0_set_clockgating_state, .set_powergating_state = gfx_v12_0_set_powergating_state, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c index 033f15e21ad33..61c3577f829fe 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c @@ -2728,20 +2728,64 @@ static void gfx_v12_1_init_golden_registers(struct amdgpu_device *adev) int i; for (i = 0; i < NUM_XCC(adev->gfx.xcc_mask); i++) { gfx_v12_1_xcc_disable_burst(adev, i); gfx_v12_1_xcc_enable_atomics(adev, i); gfx_v12_1_xcc_disable_early_write_ack(adev, i); gfx_v12_1_xcc_disable_tcp_spill_cache(adev, i); } } +static int gfx_v12_1_set_userq_eop_interrupts(struct amdgpu_device *adev, + bool enable) +{ + unsigned int irq_type; + int m, p, r; + + if (!adev->gfx.disable_kq) + return 0; + + for (m = 0; m < adev->gfx.mec.num_mec; ++m) { + for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + + p; + if (enable) + r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); + else + r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + if (r) { + if (!enable) + return r; + goto err_unwind; + } + } + } + + return 0; + +err_unwind: + for (p--; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + for (m--; m >= 0; m--) { + for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + } + return r; +} + static int gfx_v12_1_hw_init(struct amdgpu_ip_block *ip_block) { int r, i, num_xcc; struct amdgpu_device *adev = ip_block->adev; if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { /* rlc autoload firmware */ r = gfx_v12_1_rlc_backdoor_autoload_enable(adev); if (r) return r; @@ -2796,20 +2840,38 @@ static int gfx_v12_1_hw_init(struct amdgpu_ip_block *ip_block) /* * init golden registers and rlc resume may override some registers, * reconfig them here */ gfx_v12_1_tcp_harvest(adev); r = gfx_v12_1_cp_resume(adev); if (r) return r; + r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); + if (r) + return r; + + r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); + if (r) + goto err_priv_inst; + + r = gfx_v12_1_set_userq_eop_interrupts(adev, true); + if (r) + goto err_userq_eop; + + return 0; + +err_userq_eop: + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); +err_priv_inst: + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); return r; } static void gfx_v12_1_xcc_fini(struct amdgpu_device *adev, int xcc_id) { uint32_t tmp; if (!adev->no_hw_access) { if (amdgpu_gfx_disable_kcq(adev, xcc_id)) @@ -2821,55 +2883,28 @@ static void gfx_v12_1_xcc_fini(struct amdgpu_device *adev, if (amdgpu_sriov_vf(adev)) { /* Program KIQ position of RLC_CP_SCHEDULERS during destroy */ tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS); tmp &= 0xffffff00; WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp); } gfx_v12_1_xcc_cp_compute_enable(adev, false, xcc_id); gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, false, xcc_id); } -static int gfx_v12_1_set_userq_eop_interrupts(struct amdgpu_device *adev, - bool enable) -{ - unsigned int irq_type; - int m, p, r; - - if (adev->gfx.disable_kq) { - for (m = 0; m < adev->gfx.mec.num_mec; ++m) { - for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { - irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP - + (m * adev->gfx.mec.num_pipe_per_mec) - + p; - if (enable) - r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, - irq_type); - else - r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, - irq_type); - if (r) - return r; - } - } - } - - return 0; -} - static int gfx_v12_1_hw_fini(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; int i, num_xcc; - amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); - amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); gfx_v12_1_set_userq_eop_interrupts(adev, false); + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); num_xcc = NUM_XCC(adev->gfx.xcc_mask); for (i = 0; i < num_xcc; i++) { gfx_v12_1_xcc_fini(adev, i); } adev->gfxhub.funcs->gart_disable(adev); adev->gfx.is_poweron = false; @@ -2956,40 +2991,20 @@ static int gfx_v12_1_early_init(struct amdgpu_ip_block *ip_block) gfx_v12_1_set_irq_funcs(adev); gfx_v12_1_set_rlc_funcs(adev); gfx_v12_1_set_mqd_funcs(adev); gfx_v12_1_set_imu_funcs(adev); gfx_v12_1_init_rlcg_reg_access_ctrl(adev); return gfx_v12_1_init_microcode(adev); } -static int gfx_v12_1_late_init(struct amdgpu_ip_block *ip_block) -{ - struct amdgpu_device *adev = ip_block->adev; - int r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); - if (r) - return r; - - r = gfx_v12_1_set_userq_eop_interrupts(adev, true); - if (r) - return r; - - return 0; -} - static bool gfx_v12_1_is_rlc_enabled(struct amdgpu_device *adev) { uint32_t rlc_cntl; /* if RLC is not enabled, do nothing */ rlc_cntl = RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CNTL); return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false; } static void gfx_v12_1_xcc_set_safe_mode(struct amdgpu_device *adev, @@ -3869,21 +3884,20 @@ static void gfx_v12_1_emit_mem_sync(struct amdgpu_ring *ring) amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */ } static const struct amd_ip_funcs gfx_v12_1_ip_funcs = { .name = "gfx_v12_1", .early_init = gfx_v12_1_early_init, - .late_init = gfx_v12_1_late_init, .sw_init = gfx_v12_1_sw_init, .sw_fini = gfx_v12_1_sw_fini, .hw_init = gfx_v12_1_hw_init, .hw_fini = gfx_v12_1_hw_fini, .suspend = gfx_v12_1_suspend, .resume = gfx_v12_1_resume, .is_idle = gfx_v12_1_is_idle, .wait_for_idle = gfx_v12_1_wait_for_idle, .set_clockgating_state = gfx_v12_1_set_clockgating_state, .set_powergating_state = gfx_v12_1_set_powergating_state, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index bec0720f70552..47721d0c37812 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -4043,33 +4043,48 @@ static int gfx_v9_0_hw_init(struct amdgpu_ip_block *ip_block) return r; r = gfx_v9_0_cp_resume(adev); if (r) return r; if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) && !amdgpu_sriov_vf(adev)) gfx_v9_4_2_set_power_brake_sequence(adev); + r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); + if (r) + return r; + + r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); + if (r) + goto err_priv_inst; + + r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); + if (r) + goto err_bad_op; + + return 0; + +err_bad_op: + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); +err_priv_inst: + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); return r; } static int gfx_v9_0_hw_fini(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; - if (amdgpu_irq_enabled(adev, &adev->gfx.priv_reg_irq, 0)) - amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); - if (amdgpu_irq_enabled(adev, &adev->gfx.priv_inst_irq, 0)) - amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); - if (amdgpu_irq_enabled(adev, &adev->gfx.bad_op_irq, 0)) - amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); /* DF freeze and kcq disable will fail */ if (!amdgpu_ras_intr_triggered()) /* disable KCQ to avoid CPC touch memory not valid anymore */ amdgpu_gfx_disable_kcq(adev, 0); if (amdgpu_sriov_vf(adev)) { gfx_v9_0_cp_gfx_enable(adev, false); /* must disable polling for SRIOV when hw finished, otherwise * CPC engine may still keep fetching WB address which is already @@ -4856,32 +4871,20 @@ static int gfx_v9_0_ecc_late_init(struct amdgpu_ip_block *ip_block) adev->gfx.ras->enable_watchdog_timer(adev); return 0; } static int gfx_v9_0_late_init(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; int r; - r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); - if (r) - return r; - r = gfx_v9_0_ecc_late_init(ip_block); if (r) return r; if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) gfx_v9_4_2_debug_trap_config_init(adev, adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID); else gfx_v9_0_debug_trap_config_init(adev, adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c index 9f76e1af8a553..510266ba0c388 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c @@ -2364,20 +2364,38 @@ static int gfx_v9_4_3_hw_init(struct amdgpu_ip_block *ip_block) gfx_v9_4_3_constants_init(adev); r = adev->gfx.rlc.funcs->resume(adev); if (r) return r; r = gfx_v9_4_3_cp_resume(adev); if (r) return r; + r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); + if (r) + return r; + + r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); + if (r) + goto err_priv_inst; + + r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); + if (r) + goto err_bad_op; + + return 0; + +err_bad_op: + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); +err_priv_inst: + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); return r; } static int gfx_v9_4_3_perf_monitor_ptl_init(struct amdgpu_device *adev, bool enable) { struct amdgpu_ptl *ptl = &adev->psp.ptl; uint32_t ptl_state = enable ? 1 : 0; uint32_t fmt1, fmt2; int r; @@ -2439,23 +2457,23 @@ static int gfx_v9_4_3_ptl_hw_init(struct amdgpu_device *adev) } static int gfx_v9_4_3_hw_fini(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; int i, num_xcc; if (adev->psp.ptl.hw_supported && !amdgpu_in_reset(adev)) gfx_v9_4_3_perf_monitor_ptl_init(adev, false); - amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); - amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); num_xcc = NUM_XCC(adev->gfx.xcc_mask); for (i = 0; i < num_xcc; i++) { gfx_v9_4_3_xcc_fini(adev, i); } return 0; } static int gfx_v9_4_3_suspend(struct amdgpu_ip_block *ip_block) @@ -2604,33 +2622,20 @@ static int gfx_v9_4_3_early_init(struct amdgpu_ip_block *ip_block) /* init rlcg reg access ctrl */ gfx_v9_4_3_init_rlcg_reg_access_ctrl(adev); return gfx_v9_4_3_init_microcode(adev); } static int gfx_v9_4_3_late_init(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; - int r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); - if (r) - return r; if (adev->gfx.ras && adev->gfx.ras->enable_watchdog_timer) adev->gfx.ras->enable_watchdog_timer(adev); gfx_v9_4_3_ptl_hw_init(adev); return 0; } -- 2.51.2
