Re: [PATCH] drm/amdgpu: fix VFI XCC addressing for gmc v12_1 VM invalidation

[Lazar, Lijo]

On 24-Sep-26 12:31 PM, YuBiao Wang wrote:

The VM invalidation engine registers are always accessed with instance 0.
Under SR-IOV the instance is forwarded to RLCG-VFI as the target XCC, so
invalidating any GFXHUB other than the first one hits the registers of
XCC 0.

Pass the GFXHUB index as the register instance, and print the XCC ID in
the VFI access messages.

Signed-off-by: Jane Jian <jane.j...@amd.com>
Signed-off-by: YuBiao Wang <yubiao.w...@amd.com>

Reviewed-by: Lijo Lazar <lijo.la...@amd.com>

Thanks,
Lijo

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c |  8 ++--
  drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c   | 49 +++++++++++++++++++-----
  2 files changed, 44 insertions(+), 13 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c 
b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 73d9072f0c6c..faf544ed4171 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -1561,12 +1561,12 @@ static u32 amdgpu_virt_rlcg_vfi_reg_rw(struct 
amdgpu_device *adev, u32 offset, u
        spin_unlock_irqrestore(&adev->virt.rlcg_reg_lock, flags);

if (is_err)

-               dev_err(adev->dev, "VFi: [grbm_cntl=0x%x grbm_idx=0x%x] addr=0x%x 
(byte addr 0x%x), data=0x%x, cmd=0x%x\n",
-                       grbm_cntl_data, grbm_idx_data,
+               dev_err(adev->dev, "VFi: xcc%u [grbm_cntl=0x%x grbm_idx=0x%x] 
addr=0x%x (byte addr 0x%x), data=0x%x, cmd=0x%x\n",
+                       xcc_id, grbm_cntl_data, grbm_idx_data,
                        addr, addr * 4, data, cmd);
        else
-               dev_dbg(adev->dev, "VFi: [grbm_cntl=0x%x grbm_idx=0x%x] addr=0x%x 
(byte addr 0x%x), data=0x%x, cmd=0x%x\n",
-                       grbm_cntl_data, grbm_idx_data,
+               dev_dbg(adev->dev, "VFi: xcc%u [grbm_cntl=0x%x grbm_idx=0x%x] 
addr=0x%x (byte addr 0x%x), data=0x%x, cmd=0x%x\n",
+                       xcc_id, grbm_cntl_data, grbm_idx_data,
                        addr, addr * 4, data, cmd);

return data;

diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c 
b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
index cfc46f912d47..d12cdf23c836 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
@@ -292,17 +292,25 @@ static void gmc_v12_1_flush_vm_hub(struct amdgpu_device 
*adev, uint32_t vmid,
        /* Use register 17 for GART */
        const unsigned eng = 17;
        unsigned int i;
-       unsigned char hub_ip = 0;
+       u32 inst;

- hub_ip = (AMDGPU_IS_GFXHUB(vmhub)) ?

-                  GC_HWIP : MMHUB_HWIP;
+       inst = AMDGPU_IS_GFXHUB(vmhub) ? vmhub - AMDGPU_GFXHUB_START : 0;

spin_lock(&adev->gmc.invalidate_lock); if (use_semaphore) {

                for (i = 0; i < adev->usec_timeout; i++) {
                        /* a read return value of 1 means semaphore acuqire */
-                       tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem + 
hub->eng_distance * eng, hub_ip);
+                       if (AMDGPU_IS_GFXHUB(vmhub))
+                               tmp = RREG32_SOC15_IP_NO_KIQ(GC,
+                                                            
hub->vm_inv_eng0_sem +
+                                                            hub->eng_distance 
* eng,
+                                                            inst);
+                       else
+                               tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB,
+                                                            
hub->vm_inv_eng0_sem +
+                                                            hub->eng_distance 
* eng,
+                                                            inst);
                        if (tmp & 0x1)
                                break;
                        udelay(1);
@@ -312,12 +320,27 @@ static void gmc_v12_1_flush_vm_hub(struct amdgpu_device 
*adev, uint32_t vmid,
                        DRM_ERROR("Timeout waiting for sem acquire in VM 
flush!\n");
        }

- WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req, hub_ip);

+       if (AMDGPU_IS_GFXHUB(vmhub))
+               WREG32_SOC15_IP_NO_KIQ(GC,
+                                      hub->vm_inv_eng0_req + hub->eng_distance 
* eng,
+                                      inv_req, inst);
+       else
+               WREG32_SOC15_IP_NO_KIQ(MMHUB,
+                                      hub->vm_inv_eng0_req + hub->eng_distance 
* eng,
+                                      inv_req, inst);

/* Wait for ACK with a delay.*/

        for (i = 0; i < adev->usec_timeout; i++) {
-               tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_ack +
-                                   hub->eng_distance * eng, hub_ip);
+               if (AMDGPU_IS_GFXHUB(vmhub))
+                       tmp = RREG32_SOC15_IP_NO_KIQ(GC,
+                                                    hub->vm_inv_eng0_ack +
+                                                    hub->eng_distance * eng,
+                                                    inst);
+               else
+                       tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB,
+                                                    hub->vm_inv_eng0_ack +
+                                                    hub->eng_distance * eng,
+                                                    inst);
                tmp &= 1 << vmid;
                if (tmp)
                        break;
@@ -325,8 +348,16 @@ static void gmc_v12_1_flush_vm_hub(struct amdgpu_device 
*adev, uint32_t vmid,
                udelay(1);
        }

- if (use_semaphore)

-               WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem + hub->eng_distance * 
eng, 0, hub_ip);
+       if (use_semaphore) {
+               if (AMDGPU_IS_GFXHUB(vmhub))
+                       WREG32_SOC15_IP_NO_KIQ(GC,
+                                              hub->vm_inv_eng0_sem + 
hub->eng_distance * eng,
+                                              0, inst);
+               else
+                       WREG32_SOC15_IP_NO_KIQ(MMHUB,
+                                              hub->vm_inv_eng0_sem + 
hub->eng_distance * eng,
+                                              0, inst);
+       }

/* Issue additional private vm invalidation to MMHUB */

        if (!AMDGPU_IS_GFXHUB(vmhub) &&