From: Lijo Lazar <[email protected]> Add a helper to get the number of XCCs for a partition, defaulting to one when no partition manager is available.
Add the core CWSR handling infrastructure for user queues, including the first-level trap handler objects and save area sizing. Calculate the control stack, wave context and debugger memory sizes required for CWSR, and provide validation for userspace supplied save area parameters. Address validation is handled separately by the caller. Create the per-VM TMA BO using the VM root BO reservation object. This avoids recursively acquiring reservation_ww_class_mutex while the VM reservation is already held through drm_exec. The TMA BO is not pinned and is covered by the VM reservation while being added to the VM. Signed-off-by: Lijo Lazar <[email protected]> Co-developed-by: Srinivasan Shanmugam <[email protected]> Signed-off-by: Srinivasan Shanmugam <[email protected]> --- drivers/gpu/drm/amd/amdgpu/Makefile | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 3 + drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c | 534 +++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h | 81 ++++ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 19 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 13 +- drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h | 22 + 7 files changed, 660 insertions(+), 14 deletions(-) create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index 55b5d11f1938..5086a482daf1 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -72,7 +72,7 @@ amdgpu-y += amdgpu_device.o amdgpu_reg_access.o amdgpu_doorbell_mgr.o amdgpu_kms amdgpu_eeprom.o amdgpu_mca.o amdgpu_psp_ta.o amdgpu_lsdma.o amdgpu_lockdep.o \ amdgpu_ring_mux.o amdgpu_xcp.o amdgpu_seq64.o amdgpu_dev_coredump.o \ amdgpu_cper.o amdgpu_userq_fence.o amdgpu_eviction_fence.o amdgpu_ip.o \ - amdgpu_wb.o + amdgpu_wb.o amdgpu_cwsr.o amdgpu-$(CONFIG_PROC_FS) += amdgpu_fdinfo.o diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index ce671731b700..26561a4f4db8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -332,6 +332,7 @@ struct amdgpu_reset_context; struct amdgpu_reset_control; struct amdgpu_coredump_info; struct amdgpu_video_codecs; +struct amdgpu_cwsr_isa; enum amdgpu_cp_irq { AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP = 0, @@ -974,6 +975,8 @@ struct amdgpu_device { * Must be last --ends in a flexible-array member. */ struct amdgpu_kfd_dev kfd; + + struct amdgpu_cwsr_info *cwsr_info; }; static inline uint32_t amdgpu_ip_version(const struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c new file mode 100644 index 000000000000..6b6c11bb5fd7 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.c @@ -0,0 +1,534 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <drm/drm_exec.h> + +#include "amdgpu.h" +#include "cwsr_trap_handler.h" +#include "amdgpu_cwsr.h" + +extern int cwsr_enable; + +#define AMDGPU_CWSR_TBA_MAX_SIZE (2 * AMDGPU_GPU_PAGE_SIZE) +#define AMDGPU_CWSR_TMA_MAX_SIZE (AMDGPU_GPU_PAGE_SIZE) +#define AMDGPU_CWSR_TMA_OFFSET (AMDGPU_CWSR_TBA_MAX_SIZE) + +#define SGPR_SIZE_PER_CU 0x4000 +#define LDS_SIZE_PER_CU 0x10000 +#define HWREG_SIZE_PER_CU 0x1000 +#define DEBUGGER_BYTES_ALIGN 64 +#define DEBUGGER_BYTES_PER_WAVE 32 +#define SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER 40 + +enum amdgpu_cwsr_region { + AMDGPU_CWSR_TBA, + AMDGPU_CWSR_TMA, +}; + +static inline uint64_t amdgpu_cwsr_tba_vaddr(struct amdgpu_device *adev) +{ + u64 addr = AMDGPU_VA_RESERVED_TRAP_UQ_START(adev); + + addr = amdgpu_gmc_sign_extend(addr); + + return addr; +} + +static inline bool amdgpu_cwsr_is_supported(struct amdgpu_device *adev) +{ + u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0); + + if (!cwsr_enable || adev->gfx.disable_uq || + gc_ver < IP_VERSION(9, 0, 1)) + return false; + + return true; +} + +uint32_t amdgpu_cwsr_size_needed(struct amdgpu_device *adev, int num_xcc) +{ + if (!amdgpu_cwsr_is_enabled(adev)) + return 0; + + return num_xcc * + (adev->cwsr_info->xcc_cwsr_sz + adev->cwsr_info->xcc_dbg_mem_sz); +} + +static void amdgpu_cwsr_init_isa_details(struct amdgpu_device *adev, + struct amdgpu_cwsr_info *cwsr_info) +{ + u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0); + + if (gc_ver < IP_VERSION(9, 0, 1)) { + BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) > + AMDGPU_CWSR_TBA_MAX_SIZE); + cwsr_info->isa_buf = cwsr_trap_gfx8_hex; + cwsr_info->isa_sz = sizeof(cwsr_trap_gfx8_hex); + } else if (gc_ver == IP_VERSION(9, 4, 1)) { + BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex) > + AMDGPU_CWSR_TBA_MAX_SIZE); + cwsr_info->isa_buf = cwsr_trap_arcturus_hex; + cwsr_info->isa_sz = sizeof(cwsr_trap_arcturus_hex); + } else if (gc_ver == IP_VERSION(9, 4, 2)) { + BUILD_BUG_ON(sizeof(cwsr_trap_aldebaran_hex) > + AMDGPU_CWSR_TBA_MAX_SIZE); + cwsr_info->isa_buf = cwsr_trap_aldebaran_hex; + cwsr_info->isa_sz = sizeof(cwsr_trap_aldebaran_hex); + } else if (gc_ver == IP_VERSION(9, 4, 3) || + gc_ver == IP_VERSION(9, 4, 4)) { + BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_4_3_hex) > + AMDGPU_CWSR_TBA_MAX_SIZE); + cwsr_info->isa_buf = cwsr_trap_gfx9_4_3_hex; + cwsr_info->isa_sz = sizeof(cwsr_trap_gfx9_4_3_hex); + } else if (gc_ver == IP_VERSION(9, 5, 0)) { + BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_5_0_hex) > PAGE_SIZE); + cwsr_info->isa_buf = cwsr_trap_gfx9_5_0_hex; + cwsr_info->isa_sz = sizeof(cwsr_trap_gfx9_5_0_hex); + } else if (gc_ver < IP_VERSION(10, 1, 1)) { + BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) > + AMDGPU_CWSR_TBA_MAX_SIZE); + cwsr_info->isa_buf = cwsr_trap_gfx9_hex; + cwsr_info->isa_sz = sizeof(cwsr_trap_gfx9_hex); + } else if (gc_ver < IP_VERSION(10, 3, 0)) { + BUILD_BUG_ON(sizeof(cwsr_trap_nv1x_hex) > + AMDGPU_CWSR_TBA_MAX_SIZE); + cwsr_info->isa_buf = cwsr_trap_nv1x_hex; + cwsr_info->isa_sz = sizeof(cwsr_trap_nv1x_hex); + } else if (gc_ver < IP_VERSION(11, 0, 0)) { + BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex) > + AMDGPU_CWSR_TBA_MAX_SIZE); + cwsr_info->isa_buf = cwsr_trap_gfx10_hex; + cwsr_info->isa_sz = sizeof(cwsr_trap_gfx10_hex); + } else if (gc_ver < IP_VERSION(12, 0, 0)) { + /* The gfx11 cwsr trap handler must fit inside a single + page. */ + BUILD_BUG_ON(sizeof(cwsr_trap_gfx11_hex) > PAGE_SIZE); + cwsr_info->isa_buf = cwsr_trap_gfx11_hex; + cwsr_info->isa_sz = sizeof(cwsr_trap_gfx11_hex); + } else if (gc_ver < IP_VERSION(12, 1, 0)) { + BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_hex) > + AMDGPU_CWSR_TBA_MAX_SIZE); + cwsr_info->isa_buf = cwsr_trap_gfx12_hex; + cwsr_info->isa_sz = sizeof(cwsr_trap_gfx12_hex); + } else { + BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_1_0_hex) > + AMDGPU_CWSR_TBA_MAX_SIZE); + cwsr_info->isa_buf = cwsr_trap_gfx12_1_0_hex; + cwsr_info->isa_sz = sizeof(cwsr_trap_gfx12_1_0_hex); + } +} + +static uint32_t amdgpu_cwsr_get_vgpr_size_per_cu(struct amdgpu_device *adev) +{ + u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0); + u32 vgpr_size; + + switch (gc_ver) { + case IP_VERSION(9, 4, 1): /* GFX_VERSION_ARCTURUS */ + case IP_VERSION(9, 4, 2): /* GFX_VERSION_ALDEBARAN */ + case IP_VERSION(9, 4, 3): /* GFX_VERSION_AQUA_VANJARAM */ + case IP_VERSION(9, 4, 4): /* GFX_VERSION_AQUA_VANJARAM */ + case IP_VERSION(9, 5, 0): + vgpr_size = 0x80000; + break; + case IP_VERSION(11, 0, 0): + case IP_VERSION(11, 0, 2): + case IP_VERSION(11, 0, 3): + case IP_VERSION(12, 0, 0): + case IP_VERSION(12, 0, 1): + vgpr_size = 0x60000; + break; + default: + vgpr_size = 0x40000; + break; + } + + return vgpr_size; +} + +static uint32_t amdgpu_cwsr_get_wg_ctxt_size_per_cu(struct amdgpu_device *adev) +{ + u32 lds_sz_per_cu; + + lds_sz_per_cu = + (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0)) ? + (adev->gfx.cu_info.lds_size << 10) : + LDS_SIZE_PER_CU; + + return amdgpu_cwsr_get_vgpr_size_per_cu(adev) + SGPR_SIZE_PER_CU + + lds_sz_per_cu + HWREG_SIZE_PER_CU; +} + +static uint32_t amdgpu_cwsr_ctl_stack_bytes_per_wave(struct amdgpu_device *adev) +{ + u32 sz; + + if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 1, 0)) + sz = 12; + else + sz = 8; + return sz; +} + +static void amdgpu_cwsr_init_save_area_info(struct amdgpu_device *adev, + struct amdgpu_cwsr_info *cwsr_info) +{ + struct amdgpu_gfx_config *gfx_info = &adev->gfx.config; + u32 gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0); + u32 ctl_stack_size, wg_data_size, dbg_mem_size; + u32 array_count; + u32 wave_num; + u32 cu_num; + + if (gc_ver < IP_VERSION(9, 0, 1)) + return; + + array_count = gfx_info->max_shader_engines * gfx_info->max_sh_per_se; + + cu_num = adev->gfx.cu_info.number / NUM_XCC(adev->gfx.xcc_mask); + wave_num = (gc_ver < IP_VERSION(10, 1, 0)) ? /* GFX_VERSION_NAVI10 */ + min(cu_num * 40, + array_count / gfx_info->max_sh_per_se * 512) : + cu_num * 32; + + wg_data_size = ALIGN(cu_num * amdgpu_cwsr_get_wg_ctxt_size_per_cu(adev), + PAGE_SIZE); + ctl_stack_size = + wave_num * amdgpu_cwsr_ctl_stack_bytes_per_wave(adev) + 8; + ctl_stack_size = + ALIGN(SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER + ctl_stack_size, + PAGE_SIZE); + dbg_mem_size = + ALIGN(wave_num * DEBUGGER_BYTES_PER_WAVE, DEBUGGER_BYTES_ALIGN); + /* + * HW design limits control stack size to 0x7000. + * This is insufficient for theoretical PM4 cases + * but sufficient for AQL, limited by SPI events. + */ + if (IP_VERSION_MAJ(gc_ver) == 10) + ctl_stack_size = min(ctl_stack_size, 0x7000); + + cwsr_info->xcc_ctl_stack_sz = ctl_stack_size; + cwsr_info->xcc_cwsr_sz = ctl_stack_size + wg_data_size; + cwsr_info->xcc_dbg_mem_sz = dbg_mem_size; +} + +int amdgpu_cwsr_init(struct amdgpu_device *adev) +{ + struct amdgpu_cwsr_info *cwsr_info __free(kfree) = + kzalloc(sizeof(*cwsr_info), GFP_KERNEL); + void *ptr; + int r; + + if (!amdgpu_cwsr_is_supported(adev)) + return -EOPNOTSUPP; + + if (!cwsr_info) + return -ENOMEM; + amdgpu_cwsr_init_isa_details(adev, cwsr_info); + + if (!cwsr_info->isa_sz) + return -EOPNOTSUPP; + + r = amdgpu_bo_create_kernel(adev, AMDGPU_CWSR_TBA_MAX_SIZE, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_GTT, &cwsr_info->isa_bo, + NULL, &ptr); + if (r) + return r; + + memcpy(ptr, cwsr_info->isa_buf, cwsr_info->isa_sz); + + amdgpu_cwsr_init_save_area_info(adev, cwsr_info); + adev->cwsr_info = no_free_ptr(cwsr_info); + + return 0; +} + +void amdgpu_cwsr_fini(struct amdgpu_device *adev) +{ + if (!amdgpu_cwsr_is_enabled(adev)) + return; + + amdgpu_bo_free_kernel(&adev->cwsr_info->isa_bo, NULL, NULL); + kfree(adev->cwsr_info); + adev->cwsr_info = NULL; +} + +/* + * amdgpu_map_cwsr_trap_handler should be called during amdgpu_vm_init + * it maps virtual address amdgpu_cwsr_trap_handler_vaddr() to this VM, and each + * compute queue can use this virtual address for wave save/restore + * operations to support compute preemption. + */ +static int amdgpu_cwsr_map_region(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct amdgpu_cwsr_trap_obj *cwsr, + enum amdgpu_cwsr_region region) +{ + u64 cwsr_addr, va_flags, va; + struct amdgpu_bo_va **bo_va; + struct amdgpu_bo *bo; + u32 size; + int r; + + if (!cwsr || !vm) + return -EINVAL; + + cwsr_addr = amdgpu_cwsr_tba_vaddr(adev); + + if (region == AMDGPU_CWSR_TBA) { + size = AMDGPU_CWSR_TBA_MAX_SIZE; + bo_va = &cwsr->tba_va; + bo = adev->cwsr_info->isa_bo; + va = cwsr_addr; + va_flags = (AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE | + AMDGPU_VM_PAGE_EXECUTABLE); + } else { + size = AMDGPU_CWSR_TMA_MAX_SIZE; + bo_va = &cwsr->tma_va; + bo = cwsr->tma_bo; + va = cwsr_addr + AMDGPU_CWSR_TMA_OFFSET; + va_flags = (AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE); + } + + *bo_va = amdgpu_vm_bo_add(adev, vm, bo); + if (!*bo_va) + return -ENOMEM; + + va &= AMDGPU_GMC_HOLE_MASK; + r = amdgpu_vm_bo_map(adev, *bo_va, va, 0, size, va_flags); + if (r) { + dev_err(adev->dev, "failed to do bo map of %s region, err=%d\n", + (region == AMDGPU_CWSR_TBA ? "tba" : "tma"), r); + amdgpu_vm_bo_del(adev, *bo_va); + *bo_va = NULL; + return r; + } + + r = amdgpu_vm_bo_update(adev, *bo_va, false); + if (r) { + dev_err(adev->dev, + "failed to do page table update of %s region, err=%d\n", + (region == AMDGPU_CWSR_TBA ? "tba" : "tma"), r); + amdgpu_vm_bo_del(adev, *bo_va); + *bo_va = NULL; + return r; + } + + if (region == AMDGPU_CWSR_TBA) + cwsr->tba_gpu_va_addr = va; + else + cwsr->tma_gpu_va_addr = va; + + return 0; +} + +static int amdgpu_cwsr_unmap_region(struct amdgpu_device *adev, + struct amdgpu_cwsr_trap_obj *cwsr, + enum amdgpu_cwsr_region region) +{ + struct amdgpu_bo_va **bo_va; + u64 va; + int r; + + if (!cwsr) + return -EINVAL; + + if (region == AMDGPU_CWSR_TBA) { + bo_va = &cwsr->tba_va; + va = cwsr->tba_gpu_va_addr; + } else { + bo_va = &cwsr->tma_va; + va = cwsr->tma_gpu_va_addr; + } + + r = amdgpu_vm_bo_unmap(adev, *bo_va, va); + if (r) { + dev_err(adev->dev, + "failed to do bo_unmap on CWSR trap handler, err=%d\n", + r); + return r; + } + + amdgpu_vm_bo_del(adev, *bo_va); + *bo_va = NULL; + + return r; +} + +int amdgpu_cwsr_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm, + struct amdgpu_cwsr_trap_obj **trap_obj) +{ + struct amdgpu_bo_param bp; + struct amdgpu_cwsr_trap_obj *cwsr; + struct amdgpu_bo *bo; + struct drm_exec exec; + int r; + + if (!amdgpu_cwsr_is_enabled(adev)) + return -EOPNOTSUPP; + if (!vm || !trap_obj) + return -EINVAL; + + cwsr = kzalloc_obj(*cwsr, GFP_KERNEL); + if (!cwsr) + return -ENOMEM; + + bo = adev->cwsr_info->isa_bo; + drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0); + drm_exec_until_all_locked(&exec) { + r = amdgpu_vm_lock_pd(vm, &exec, 0); + if (likely(!r)) + r = drm_exec_lock_obj(&exec, &bo->tbo.base); + drm_exec_retry_on_contention(&exec); + if (unlikely(r)) { + dev_err(adev->dev, + "failed to reserve for CWSR allocs: err=%d\n", + r); + goto err_exec; + } + } + + /* + * Create TMA BO using the VM's already-locked resv so no new + * reservation_ww_class_mutex acquisition is needed, avoiding the + * recursive ww_mutex deadlock. Unpinned; eviction is handled via + * amdgpu_vm_bo_add() inside amdgpu_cwsr_map_region(). + */ + memset(&bp, 0, sizeof(bp)); + bp.size = AMDGPU_CWSR_TMA_MAX_SIZE; + bp.byte_align = PAGE_SIZE; + bp.domain = AMDGPU_GEM_DOMAIN_GTT; + bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC | + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + bp.type = ttm_bo_type_kernel; + bp.bo_ptr_size = sizeof(struct amdgpu_bo); + bp.resv = vm->root.bo->tbo.base.resv; + + r = amdgpu_bo_create(adev, &bp, &cwsr->tma_bo); + if (r) + goto err_exec; + + /* VM resv already held, kmap directly, no separate reserve needed. */ + r = amdgpu_bo_kmap(cwsr->tma_bo, &cwsr->tma_cpu_addr); + if (r) + goto err_tma; + + memset(cwsr->tma_cpu_addr, 0, AMDGPU_CWSR_TMA_MAX_SIZE); + + r = amdgpu_cwsr_map_region(adev, vm, cwsr, AMDGPU_CWSR_TMA); + if (r) + goto err_kmap; + + r = amdgpu_cwsr_map_region(adev, vm, cwsr, AMDGPU_CWSR_TBA); + if (r) { + amdgpu_cwsr_unmap_region(adev, cwsr, AMDGPU_CWSR_TMA); + goto err_kmap; + } + + drm_exec_fini(&exec); + *trap_obj = cwsr; + return 0; + +err_kmap: + amdgpu_bo_kunmap(cwsr->tma_bo); +err_tma: + amdgpu_bo_unref(&cwsr->tma_bo); +err_exec: + drm_exec_fini(&exec); + kfree(cwsr); + *trap_obj = NULL; + return r; +} + +int amdgpu_cwsr_validate_params(struct amdgpu_device *adev, + struct amdgpu_cwsr_params *cwsr_params, + int num_xcc) +{ + u32 min_size, max_size; + + if (!amdgpu_cwsr_is_enabled(adev)) + return -EOPNOTSUPP; + + if (!cwsr_params) + return -EINVAL; + + min_size = amdgpu_cwsr_size_needed(adev, num_xcc); + max_size = 2 * min_size; + /* + * Only save area size details checked. Address validation needs to be + * carried out separately. User is expected to pass a cwsr buffer area + * with size >= ctl stack size + wg data size + debug memory size + */ + if (cwsr_params->cwsr_sz < min_size || + cwsr_params->cwsr_sz > max_size) { + dev_dbg(adev->dev, + "queue cwsr size 0x%x should be within min %u : max %u\n", + cwsr_params->cwsr_sz, min_size, max_size); + return -EINVAL; + } + + return 0; +} + +void amdgpu_cwsr_free(struct amdgpu_device *adev, struct amdgpu_vm *vm, + struct amdgpu_cwsr_trap_obj **trap_obj) +{ + struct amdgpu_bo *tba_bo; + struct amdgpu_bo *tma_bo; + struct drm_exec exec; + int r; + + if (!trap_obj || !*trap_obj || !(*trap_obj)->tma_bo) + return; + + tba_bo = adev->cwsr_info->isa_bo; + tma_bo = (*trap_obj)->tma_bo; + + if (!tba_bo || !tma_bo) + return; + + drm_exec_init(&exec, 0, 0); + drm_exec_until_all_locked(&exec) { + r = amdgpu_vm_lock_pd(vm, &exec, 0); + if (likely(!r)) + r = drm_exec_lock_obj(&exec, &tba_bo->tbo.base); + drm_exec_retry_on_contention(&exec); + if (unlikely(r)) { + dev_err(adev->dev, + "failed to reserve CWSR BOs: err=%d\n", r); + goto err; + } + } + + amdgpu_cwsr_unmap_region(adev, *trap_obj, AMDGPU_CWSR_TBA); + amdgpu_cwsr_unmap_region(adev, *trap_obj, AMDGPU_CWSR_TMA); + +err: + drm_exec_fini(&exec); + + /* TMA BO not pinned, release CPU mapping and free. */ + amdgpu_bo_kunmap((*trap_obj)->tma_bo); + amdgpu_bo_unref(&(*trap_obj)->tma_bo); + kfree(*trap_obj); + *trap_obj = NULL; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h new file mode 100644 index 000000000000..7f922559e297 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cwsr.h @@ -0,0 +1,81 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef AMDGPU_CWSR_H +#define AMDGPU_CWSR_H + +#include <linux/types.h> + +struct amdgpu_bo; +struct amdgpu_bo_va; +struct amdgpu_device; +struct amdgpu_vm; + +/** + * struct amdgpu_cwsr_obj - CWSR (Compute Wave Save Restore) buffer tracking + * @bo: Buffer object for CWSR area + * @bo_va: Buffer object virtual address mapping + */ +struct amdgpu_cwsr_trap_obj { + u64 tma_gpu_va_addr; + u64 tba_gpu_va_addr; + + struct amdgpu_bo *tma_bo; + struct amdgpu_bo_va *tba_va; + struct amdgpu_bo_va *tma_va; + void *tma_cpu_addr; +}; + +struct amdgpu_cwsr_info { + /* cwsr isa */ + struct amdgpu_bo *isa_bo; + const void *isa_buf; + u32 isa_sz; + /* cwsr size info per XCC*/ + u32 xcc_ctl_stack_sz; + u32 xcc_dbg_mem_sz; + u32 xcc_cwsr_sz; +}; + +struct amdgpu_cwsr_params { + u64 ctx_save_area_address; + /* cwsr size info */ + u32 cwsr_sz; +}; + +int amdgpu_cwsr_init(struct amdgpu_device *adev); +void amdgpu_cwsr_fini(struct amdgpu_device *adev); + +int amdgpu_cwsr_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm, + struct amdgpu_cwsr_trap_obj **cwsr_obj); +void amdgpu_cwsr_free(struct amdgpu_device *adev, struct amdgpu_vm *vm, + struct amdgpu_cwsr_trap_obj **cwsr_obj); +static inline bool amdgpu_cwsr_is_enabled(struct amdgpu_device *adev) +{ + return adev->cwsr_info; +} + +uint32_t amdgpu_cwsr_size_needed(struct amdgpu_device *adev, int num_xcc); +int amdgpu_cwsr_validate_params(struct amdgpu_device *adev, + struct amdgpu_cwsr_params *cwsr_params, + int num_xcc); +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index ab6f9131ac11..b2a8a8b37006 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -764,6 +764,15 @@ MODULE_PARM_DESC(use_xgmi_p2p, "Enable XGMI P2P interface (0 = disable; 1 = enable (default))"); module_param_named(use_xgmi_p2p, amdgpu_use_xgmi_p2p, int, 0444); +/** + * DOC: cwsr_enable (int) + * CWSR(compute wave store and resume) allows the GPU to preempt shader execution in + * the middle of a compute wave. Default is 1 to enable this feature. Setting 0 + * disables it. + */ +int cwsr_enable = 1; +module_param(cwsr_enable, int, 0444); +MODULE_PARM_DESC(cwsr_enable, "CWSR enable (0 = Off, 1 = On (Default))"); #ifdef CONFIG_HSA_AMD /** @@ -787,16 +796,6 @@ module_param(hws_max_conc_proc, int, 0444); MODULE_PARM_DESC(hws_max_conc_proc, "Max # processes HWS can execute concurrently when sched_policy=0 (0 = no concurrency, #VMIDs for KFD = Maximum(default))"); -/** - * DOC: cwsr_enable (int) - * CWSR(compute wave store and resume) allows the GPU to preempt shader execution in - * the middle of a compute wave. Default is 1 to enable this feature. Setting 0 - * disables it as only in non-HWS mode. - */ -int cwsr_enable = 1; -module_param(cwsr_enable, int, 0444); -MODULE_PARM_DESC(cwsr_enable, "CWSR enable (0 = Off (debugging only), 1 = On (Default))"); - /** * DOC: max_num_of_queues_per_device (int) * Maximum number of queues per device. Valid setting is between 1 and 4096. Default diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index 8e478d3bda1a..226caa99f6ee 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -178,10 +178,17 @@ struct amdgpu_bo_vm; #define AMDGPU_VA_RESERVED_TRAP_SIZE (1ULL << 16) #define AMDGPU_VA_RESERVED_TRAP_START(adev) (AMDGPU_VA_RESERVED_SEQ64_START(adev) \ - AMDGPU_VA_RESERVED_TRAP_SIZE) +/* TBD: Ideally, existing TRAP VA should do. There is a conflict with KFD mapping that needs to be + * resolved. Revisit later. + */ +#define AMDGPU_VA_RESERVED_TRAP_UQ_SIZE (3ULL << 12) +#define AMDGPU_VA_RESERVED_TRAP_UQ_START(adev) \ + (AMDGPU_VA_RESERVED_TRAP_START(adev) - AMDGPU_VA_RESERVED_TRAP_UQ_SIZE) + #define AMDGPU_VA_RESERVED_BOTTOM (1ULL << 16) -#define AMDGPU_VA_RESERVED_TOP (AMDGPU_VA_RESERVED_TRAP_SIZE + \ - AMDGPU_VA_RESERVED_SEQ64_SIZE + \ - AMDGPU_VA_RESERVED_CSA_SIZE) +#define AMDGPU_VA_RESERVED_TOP \ + (AMDGPU_VA_RESERVED_TRAP_UQ_SIZE + AMDGPU_VA_RESERVED_TRAP_SIZE + \ + AMDGPU_VA_RESERVED_SEQ64_SIZE + AMDGPU_VA_RESERVED_CSA_SIZE) /* See vm_update_mode */ #define AMDGPU_VM_USE_CPU_FOR_GFX (1 << 0) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h index 33157409eda0..e7c84b50ccf4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h @@ -226,4 +226,26 @@ amdgpu_get_next_xcp(struct amdgpu_xcp_mgr *xcp_mgr, int *from) for (i = 0, xcp = amdgpu_get_next_xcp(xcp_mgr, &i); xcp; \ ++i, xcp = amdgpu_get_next_xcp(xcp_mgr, &i)) +static inline int amdgpu_xcp_get_num_xcc(struct amdgpu_xcp_mgr *xcp_mgr, + int xcp_id) +{ + struct amdgpu_xcp *xcp; + u32 xcc_mask; + int i, r; + + if (!xcp_mgr || xcp_id == AMDGPU_XCP_NO_PARTITION) + return 1; + for_each_xcp(xcp_mgr, xcp, i) { + if (xcp->id == xcp_id) { + r = amdgpu_xcp_get_inst_details(xcp, AMDGPU_XCP_GFX, + &xcc_mask); + if (unlikely(r)) + return 1; + else + return hweight32(xcc_mask); + } + } + + return 1; +} #endif -- 2.34.1
