From: Mukul Joshi <[email protected]> Rework the VMID reservation logic by passing in the mask of VMIDs (for both GFXHUB and MMHUB) available for it to use. The VMID masks for both GFXHUB and MMHUB are initialized in the respective ASIC specific files. This also helps in reserving per-ASIC VMIDs. This is in preparation for reserving a VMID for NPA added in the next patch.
Signed-off-by: Mukul Joshi <[email protected]> Reviewed-by: Christian König <[email protected]> Signed-off-by: Alex Deucher <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c | 33 ++++++++++++++----------- drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h | 27 +++++++++++++++++++- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c | 2 +- drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 4 +++ drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c | 5 ++++ drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c | 14 +++++++---- drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c | 6 +++++ drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c | 6 +++++ drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c | 6 +++++ drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 6 +++++ 17 files changed, 95 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c index 8a2d64f0ebc3f..8c32c69437645 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c @@ -625,12 +625,27 @@ void amdgpu_vmid_reset_all(struct amdgpu_device *adev) for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[i]; - - for (j = 1; j < id_mgr->num_ids; ++j) + for_each_set_bit(j, id_mgr->vmid_mask, AMDGPU_NUM_VMID) amdgpu_vmid_reset(adev, i, j); } } +void amdgpu_vmid_mgr_set_vmid_mask(struct amdgpu_device *adev, + unsigned long vmid_mask, bool for_mmhub) +{ + unsigned int i; + + BUILD_BUG_ON(AMDGPU_NUM_VMID > BITS_PER_LONG); + + for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) { + bool is_mmhub = AMDGPU_IS_MMHUB0(i) || AMDGPU_IS_MMHUB1(i); + + if (is_mmhub == for_mmhub) + bitmap_copy(adev->vm_manager.id_mgr[i].vmid_mask, + &vmid_mask, AMDGPU_NUM_VMID); + } +} + /** * amdgpu_vmid_mgr_init - init the VMID manager * @@ -649,19 +664,8 @@ void amdgpu_vmid_mgr_init(struct amdgpu_device *adev) mutex_init(&id_mgr->lock); INIT_LIST_HEAD(&id_mgr->ids_lru); - /* for GC <10, SDMA uses MMHUB so use first_kfd_vmid for both GC and MM */ - if (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(10, 0, 0)) - /* manage only VMIDs not used by KFD */ - id_mgr->num_ids = adev->vm_manager.first_kfd_vmid; - else if (AMDGPU_IS_MMHUB0(i) || - AMDGPU_IS_MMHUB1(i)) - id_mgr->num_ids = 16; - else - /* manage only VMIDs not used by KFD */ - id_mgr->num_ids = adev->vm_manager.first_kfd_vmid; - /* skip over VMID 0, since it is the system VM */ - for (j = 1; j < id_mgr->num_ids; ++j) { + for_each_set_bit(j, id_mgr->vmid_mask, AMDGPU_NUM_VMID) { amdgpu_vmid_reset(adev, i, j); amdgpu_sync_create(&id_mgr->ids[j].active); list_add_tail(&id_mgr->ids[j].list, &id_mgr->ids_lru); @@ -685,6 +689,7 @@ void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev) &adev->vm_manager.id_mgr[i]; mutex_destroy(&id_mgr->lock); + for (j = 0; j < AMDGPU_NUM_VMID; ++j) { struct amdgpu_vmid *id = &id_mgr->ids[j]; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h index 4b55d0d9703a3..46b2c21601263 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h @@ -26,6 +26,8 @@ #include <linux/types.h> #include <linux/mutex.h> #include <linux/list.h> +#include <linux/bitops.h> +#include <linux/find.h> #include <linux/dma-fence.h> #include "amdgpu_sync.h" @@ -33,6 +35,27 @@ /* maximum number of VMIDs */ #define AMDGPU_NUM_VMID 16 +/** + * for_each_vmid_and_zero - iterate over VMID 0 plus every VMID set in a hub's + * vmid_mask + * @vmid: loop cursor (int / unsigned int) + * @adev: struct amdgpu_device * + * @hub: vmhub index into adev->vm_manager.id_mgr[] + * + * Equivalent to OR-ing BIT(0) into id_mgr[hub].vmid_mask and walking the + * result with for_each_set_bit(). Used by the gfx_v*_constants_init() paths + * which must always touch VMID 0 (system) in addition to the regular VMIDs + * owned by the hub. + * + * Relies on AMDGPU_NUM_VMID (16) fitting in a single unsigned long, i.e. + * vmid_mask[] being a 1-element bitmap. + */ +#define for_each_vmid_and_zero(vmid, adev, hub) \ + for (unsigned long __vmid_mask = \ + (adev)->vm_manager.id_mgr[(hub)].vmid_mask[0] | BIT(0); \ + __vmid_mask; __vmid_mask = 0) \ + for_each_set_bit((vmid), &__vmid_mask, AMDGPU_NUM_VMID) + struct amdgpu_device; struct amdgpu_fpriv; struct amdgpu_vm; @@ -65,10 +88,10 @@ struct amdgpu_vmid { struct amdgpu_vmid_mgr { struct mutex lock; - unsigned num_ids; struct list_head ids_lru; struct amdgpu_vmid ids[AMDGPU_NUM_VMID]; bool reserved_vmid; + DECLARE_BITMAP(vmid_mask, AMDGPU_NUM_VMID); }; int amdgpu_pasid_alloc(unsigned int bits, struct amdgpu_fpriv *fpriv); @@ -95,5 +118,7 @@ void amdgpu_vmid_reset_all(struct amdgpu_device *adev); void amdgpu_vmid_mgr_init(struct amdgpu_device *adev); void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev); +void amdgpu_vmid_mgr_set_vmid_mask(struct amdgpu_device *adev, + unsigned long vmid_mask, bool for_mmhub); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index 5033f85d31022..045b9c7d2a835 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -5368,7 +5368,7 @@ static void gfx_v10_0_constants_init(struct amdgpu_device *adev) /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { nv_grbm_select(adev, 0, 0, 0, i); /* CP and shaders */ WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index d704ce328e6ec..e5dbd71cce763 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -2217,7 +2217,7 @@ static void gfx_v11_0_constants_init(struct amdgpu_device *adev) /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { soc21_grbm_select(adev, 0, 0, 0, i); /* CP and shaders */ WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c index df8d36be613ac..6bb1c42efc5e8 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c @@ -1856,7 +1856,7 @@ static void gfx_v12_0_constants_init(struct amdgpu_device *adev) /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { soc24_grbm_select(adev, 0, 0, 0, i); /* CP and shaders */ WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c index 73fc18ec94a20..d60cd64d444eb 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c @@ -1732,7 +1732,7 @@ static void gfx_v12_1_xcc_constants_init(struct amdgpu_device *adev, /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { soc_v1_0_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id)); /* CP and shaders */ WREG32_SOC15(GC, GET_INST(GC, xcc_id), diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c index 755371b665746..549d1239e326b 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c @@ -1938,7 +1938,7 @@ static void gfx_v7_0_constants_init(struct amdgpu_device *adev) WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg); mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) { + for_each_vmid_and_zero(i, adev, 0) { if (i == 0) sh_mem_base = 0; else diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 63288382bcf40..950874ef8d045 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -3765,7 +3765,7 @@ static void gfx_v8_0_constants_init(struct amdgpu_device *adev) WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg); mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) { + for_each_vmid_and_zero(i, adev, 0) { vi_srbm_select(adev, 0, 0, 0, i); /* CP and shaders */ if (i == 0) { diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 0dae8ff9c762a..3ba2584e48c6a 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -2686,7 +2686,7 @@ static void gfx_v9_0_constants_init(struct amdgpu_device *adev) /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { soc15_grbm_select(adev, 0, 0, 0, i, 0); /* CP and shaders */ if (i == 0) { diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c index e1603563c49de..ce48da12edb29 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c @@ -1241,7 +1241,7 @@ static void gfx_v9_4_3_xcc_constants_init(struct amdgpu_device *adev, /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { soc15_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id)); /* CP and shaders */ if (i == 0) { diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c index 75a552782c979..9a8a7921ce982 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c @@ -869,6 +869,10 @@ static int gmc_v10_0_sw_init(struct amdgpu_ip_block *ip_block) * amdkfd will use VMIDs 8-15 */ adev->vm_manager.first_kfd_vmid = 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(AMDGPU_NUM_VMID - 1, 1), true); amdgpu_vm_manager_init(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c index 7fb97be16b7c1..bee269a8877a5 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c @@ -855,6 +855,11 @@ static int gmc_v11_0_sw_init(struct amdgpu_ip_block *ip_block) */ adev->vm_manager.first_kfd_vmid = adev->gfx.disable_kq ? 1 : 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(AMDGPU_NUM_VMID - 1, 1), true); + amdgpu_vm_manager_init(adev); r = amdgpu_gmc_ras_sw_init(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c index 18511e7995ef9..3ae3ba42c6b0f 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c @@ -957,13 +957,17 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block) * number of VMs * VMID 0 is reserved for System * amdgpu graphics/compute will use VMIDs 1-7 - * amdkfd will use VMIDs 8-15 + * amdkfd will use VMIDs 8-15. + * On GFX 12.1, amdkfd will use VMIDs 3-15. */ adev->vm_manager.first_kfd_vmid = - amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) ? - 3 : 8; - adev->vm_manager.first_kfd_vmid = - adev->gfx.disable_kq ? 1 : (adev->vm_manager.first_kfd_vmid); + adev->gfx.disable_kq ? 1 : + (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) ? + 3 : 8); + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(AMDGPU_NUM_VMID - 1, 1), true); amdgpu_vm_manager_init(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c index ca7afb9b7d0ae..cdd11587b12ae 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c @@ -864,6 +864,12 @@ static int gmc_v6_0_sw_init(struct amdgpu_ip_block *ip_block) * amdkfd will use VMIDs 8-15 */ adev->vm_manager.first_kfd_vmid = 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + true); amdgpu_vm_manager_init(adev); /* base offset of vram pages */ diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c index 1c1eccfe383fd..c9cc55e18893b 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c @@ -1045,6 +1045,12 @@ static int gmc_v7_0_sw_init(struct amdgpu_ip_block *ip_block) * amdkfd will use VMIDs 8-15 */ adev->vm_manager.first_kfd_vmid = 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + true); amdgpu_vm_manager_init(adev); /* base offset of vram pages */ diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c index bf089595e5dac..8d8a0775878ff 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c @@ -1122,6 +1122,12 @@ static int gmc_v8_0_sw_init(struct amdgpu_ip_block *ip_block) * amdkfd will use VMIDs 8-15 */ adev->vm_manager.first_kfd_vmid = 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + true); amdgpu_vm_manager_init(adev); /* base offset of vram pages */ diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index b46b87291c512..633772fa71fda 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -2038,6 +2038,12 @@ static int gmc_v9_0_sw_init(struct amdgpu_ip_block *ip_block) 3 : 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + true); amdgpu_vm_manager_init(adev); gmc_v9_0_save_registers(adev); -- 2.55.0
