On the A+A VF the hypervisor reports each VF's VRAM address and length in the ACPI SRAT table. The XGMI LFB region and segment registers describe the physical GPU, not the VF, so the node id and segment read from those registers are not a valid VRAM starting address.
Read the SRAT length in mmhub_v4_2_0_get_xgmi_info() and report the VF as one physical node at id 0 whose segment is that length. get_mc_fb_offset() reads the SRAT address on each call. mc_init copies that address, plus the node product, into aper_base and sets aper_size from the segment. Node id 0 makes the product zero, so the PTE base stays at the SRAT address. v2: - Explain that the SRAT address is already the VF VRAM base and must not be adjusted by the XGMI node segment. v3: - Report the VF as XGMI node 0 of 1 instead of special-casing the node offset and the PDB0 size. v4: - Read SRAT in get_xgmi_info() and derive aper_base and aper_size from the FB offset and the segment size. - Move the 8GB GART alignment into its own patch. v5: - Revert the SRAT aperture commit with the other two reverts. Leave mc_init on the restored connected_to_cpu path. v6: - Read the SRAT address in both get_mc_fb_offset() helpers on each call instead of caching it. Keep the SRAT length as the segment size. - PDB0 is already enabled because the VF is CPU-connected. Leave amdgpu_gmc_is_pdb0_enabled() unchanged. Signed-off-by: Zhigang Luo <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h | 8 +++-- drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c | 16 ++++++++++ drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c | 38 +++++++++++++++++++++++ 4 files changed, 60 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h index 5f36aff17e79e..47092ec6dff2c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h @@ -79,11 +79,13 @@ struct amdgpu_xgmi { /* from psp */ u64 node_id; u64 hive_id; - /* fixed per family */ + /* Segment size. Fixed per family on a hive. On an SPM VF this is + * the SRAT length of that one VF. + */ u64 node_segment_size; - /* physical node (0-3) */ + /* Physical node, 0-3. An SPM VF is node 0. */ unsigned physical_node_id; - /* number of nodes (0-4) */ + /* Number of nodes, 0-4. An SPM VF reports one node. */ unsigned num_physical_nodes; /* gpu list in the same hive */ struct list_head head; diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c index b13967a1c8fba..837ad28530e48 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c @@ -53,6 +53,22 @@ static u64 gfxhub_v12_1_get_fb_location(struct amdgpu_device *adev) static u64 gfxhub_v12_1_get_mc_fb_offset(struct amdgpu_device *adev) { + /* SPM VRAM starts at the SRAT address. */ + if (amdgpu_virt_vram_is_spm(adev)) { + u64 fb_offset, seg_size; + int r; + + r = amdgpu_acpi_find_gpu_memory_in_srat(adev, &fb_offset, + &seg_size); + if (r) { + dev_err(adev->dev, + "Failed to find SPM VRAM in SRAT: %d\n", r); + return 0; + } + + return fb_offset; + } + return (u64)(RREG32_SOC15(GC, GET_INST(GC, 0), regGCMC_VM_FB_OFFSET) << 24); } diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c index 7e0b83b1015fc..14ef841d70365 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c @@ -707,7 +707,7 @@ static void gmc_v12_0_vram_gtt_location(struct amdgpu_device *adev, amdgpu_gmc_agp_location(adev, mc); } /* base offset of vram pages */ - if (amdgpu_sriov_vf(adev)) + if (amdgpu_sriov_vf(adev) && !amdgpu_virt_vram_is_spm(adev)) adev->vm_manager.vram_base_offset = 0; else adev->vm_manager.vram_base_offset = adev->mmhub.funcs->get_mc_fb_offset(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c index 4495d52b4b07c..ba4483948ce5f 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c @@ -132,6 +132,26 @@ static int mmhub_v4_2_0_get_xgmi_info(struct amdgpu_device *adev) if (!adev->gmc.xgmi.connected_to_cpu) return 0; + /* The SRAT entry is the VF's whole VRAM. PF_LFB_REGION describes + * the physical GPU, not the VF, so report a single node. The + * length is the segment size. get_mc_fb_offset() reads the + * address later. + */ + if (amdgpu_virt_vram_is_spm(adev)) { + u64 fb_offset; + int r; + + r = amdgpu_acpi_find_gpu_memory_in_srat(adev, &fb_offset, + &seg_size); + if (r) + return r; + + adev->gmc.xgmi.num_physical_nodes = 1; + adev->gmc.xgmi.physical_node_id = 0; + adev->gmc.xgmi.node_segment_size = seg_size; + return 0; + } + xgmi_lfb_cntl = RREG32_SOC15(MMHUB, GET_INST(MMHUB, 0), regMMMC_VM_XGMI_LFB_CNTL); seg_size = REG_GET_FIELD( @@ -177,6 +197,24 @@ static u64 mmhub_v4_2_0_get_fb_location(struct amdgpu_device *adev) static u64 mmhub_v4_2_0_get_mc_fb_offset(struct amdgpu_device *adev) { + /* SPM VRAM starts at the SRAT address. mc_init copies it into + * aper_base. + */ + if (amdgpu_virt_vram_is_spm(adev)) { + u64 fb_offset, seg_size; + int r; + + r = amdgpu_acpi_find_gpu_memory_in_srat(adev, &fb_offset, + &seg_size); + if (r) { + dev_err(adev->dev, + "Failed to find SPM VRAM in SRAT: %d\n", r); + return 0; + } + + return fb_offset; + } + return (u64)RREG32_SOC15(MMHUB, GET_INST(MMHUB, 0), regMMMC_VM_FB_OFFSET) << 24; } -- 2.43.0
