Avoid constant register reloads while emitting IBs by using a local write pointer and only updating the size at the end of each helper.
Signed-off-by: Tvrtko Ursulin <[email protected]> Cc: Timur Kristóf <[email protected]> Cc: Alex Deucher <[email protected]> --- drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 115 +++++++++++++------------- 1 file changed, 58 insertions(+), 57 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 3ba2584e48c6..60f410ab1afa 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -4703,6 +4703,7 @@ static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) int vgpr_init_shader_size; const u32 *vgpr_init_shader_ptr; const struct soc15_reg_entry *vgpr_init_regs_ptr; + u32 *ptr; /* only support when RAS is enabled */ if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) @@ -4746,98 +4747,98 @@ static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) ib = &job->ibs[0]; /* load the compute shaders */ - for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++) - ib->ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i]; + memcpy(&ib->ptr[vgpr_offset / sizeof(u32)], vgpr_init_shader_ptr, + vgpr_init_shader_size); + memcpy(&ib->ptr[sgpr_offset / sizeof(u32)], sgpr_init_compute_shader, + sizeof(sgpr_init_compute_shader)); - for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) - ib->ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; - - /* init the ib length to 0 */ - ib->length_dw = 0; + ptr = ib->ptr; /* VGPR */ /* write the register state for the compute dispatch */ for (i = 0; i < gpr_reg_size; i++) { - ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); - ib->ptr[ib->length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i]) - - PACKET3_SET_SH_REG_START; - ib->ptr[ib->length_dw++] = vgpr_init_regs_ptr[i].reg_value; + *ptr++ = PACKET3(PACKET3_SET_SH_REG, 1); + *ptr++ = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i]) - + PACKET3_SET_SH_REG_START; + *ptr++ = vgpr_init_regs_ptr[i].reg_value; } /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ gpu_addr = (ib->gpu_addr + (u64)vgpr_offset) >> 8; - ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); - ib->ptr[ib->length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) - - PACKET3_SET_SH_REG_START; - ib->ptr[ib->length_dw++] = lower_32_bits(gpu_addr); - ib->ptr[ib->length_dw++] = upper_32_bits(gpu_addr); + *ptr++ = PACKET3(PACKET3_SET_SH_REG, 2); + *ptr++ = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) - + PACKET3_SET_SH_REG_START; + *ptr++ = lower_32_bits(gpu_addr); + *ptr++ = upper_32_bits(gpu_addr); /* write dispatch packet */ - ib->ptr[ib->length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); - ib->ptr[ib->length_dw++] = compute_dim_x * 2; /* x */ - ib->ptr[ib->length_dw++] = 1; /* y */ - ib->ptr[ib->length_dw++] = 1; /* z */ - ib->ptr[ib->length_dw++] = - REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); + *ptr++ = PACKET3(PACKET3_DISPATCH_DIRECT, 3); + *ptr++ = compute_dim_x * 2; /* x */ + *ptr++ = 1; /* y */ + *ptr++ = 1; /* z */ + *ptr++ = REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, + 1); /* write CS partial flush packet */ - ib->ptr[ib->length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); - ib->ptr[ib->length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); + *ptr++ = PACKET3(PACKET3_EVENT_WRITE, 0); + *ptr++ = EVENT_TYPE(7) | EVENT_INDEX(4); /* SGPR1 */ /* write the register state for the compute dispatch */ for (i = 0; i < gpr_reg_size; i++) { - ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); - ib->ptr[ib->length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i]) - - PACKET3_SET_SH_REG_START; - ib->ptr[ib->length_dw++] = sgpr1_init_regs[i].reg_value; + *ptr++ = PACKET3(PACKET3_SET_SH_REG, 1); + *ptr++ = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i]) - + PACKET3_SET_SH_REG_START; + *ptr++ = sgpr1_init_regs[i].reg_value; } /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ gpu_addr = (ib->gpu_addr + (u64)sgpr_offset) >> 8; - ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); - ib->ptr[ib->length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) - - PACKET3_SET_SH_REG_START; - ib->ptr[ib->length_dw++] = lower_32_bits(gpu_addr); - ib->ptr[ib->length_dw++] = upper_32_bits(gpu_addr); + *ptr++ = PACKET3(PACKET3_SET_SH_REG, 2); + *ptr++ = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) - + PACKET3_SET_SH_REG_START; + *ptr++ = lower_32_bits(gpu_addr); + *ptr++ = upper_32_bits(gpu_addr); /* write dispatch packet */ - ib->ptr[ib->length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); - ib->ptr[ib->length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ - ib->ptr[ib->length_dw++] = 1; /* y */ - ib->ptr[ib->length_dw++] = 1; /* z */ - ib->ptr[ib->length_dw++] = - REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); + *ptr++ = PACKET3(PACKET3_DISPATCH_DIRECT, 3); + *ptr++ = compute_dim_x / 2 * sgpr_work_group_size; /* x */ + *ptr++ = 1; /* y */ + *ptr++ = 1; /* z */ + *ptr++ = REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, + 1); /* write CS partial flush packet */ - ib->ptr[ib->length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); - ib->ptr[ib->length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); + *ptr++ = PACKET3(PACKET3_EVENT_WRITE, 0); + *ptr++ = EVENT_TYPE(7) | EVENT_INDEX(4); /* SGPR2 */ /* write the register state for the compute dispatch */ for (i = 0; i < gpr_reg_size; i++) { - ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); - ib->ptr[ib->length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i]) - - PACKET3_SET_SH_REG_START; - ib->ptr[ib->length_dw++] = sgpr2_init_regs[i].reg_value; + *ptr++ = PACKET3(PACKET3_SET_SH_REG, 1); + *ptr++ = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i]) - + PACKET3_SET_SH_REG_START; + *ptr++ = sgpr2_init_regs[i].reg_value; } /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ gpu_addr = (ib->gpu_addr + (u64)sgpr_offset) >> 8; - ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); - ib->ptr[ib->length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) + *ptr++ = PACKET3(PACKET3_SET_SH_REG, 2); + *ptr++ = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) - PACKET3_SET_SH_REG_START; - ib->ptr[ib->length_dw++] = lower_32_bits(gpu_addr); - ib->ptr[ib->length_dw++] = upper_32_bits(gpu_addr); + *ptr++ = lower_32_bits(gpu_addr); + *ptr++ = upper_32_bits(gpu_addr); /* write dispatch packet */ - ib->ptr[ib->length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); - ib->ptr[ib->length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ - ib->ptr[ib->length_dw++] = 1; /* y */ - ib->ptr[ib->length_dw++] = 1; /* z */ - ib->ptr[ib->length_dw++] = - REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); + *ptr++ = PACKET3(PACKET3_DISPATCH_DIRECT, 3); + *ptr++ = compute_dim_x / 2 * sgpr_work_group_size; /* x */ + *ptr++ = 1; /* y */ + *ptr++ = 1; /* z */ + *ptr++ = REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, + 1); /* write CS partial flush packet */ - ib->ptr[ib->length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); - ib->ptr[ib->length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); + *ptr++ = PACKET3(PACKET3_EVENT_WRITE, 0); + *ptr++ = EVENT_TYPE(7) | EVENT_INDEX(4); + + ib->length_dw = ptr - ib->ptr; /* schedule the ib on the ring */ r = amdgpu_job_submit_direct(job, ring, &f); -- 2.55.0
