The buddy allocator manages the actual usable VRAM. On my GA102 Ampere with 24GB video memory, that is ~23.7GB on a 24GB GPU enabling proper GPU memory allocation for driver use.
Signed-off-by: Joel Fernandes <[email protected]> --- drivers/gpu/nova-core/gpu.rs | 62 ++++++++++++++++++++++----- drivers/gpu/nova-core/gsp/boot.rs | 7 ++- drivers/gpu/nova-core/gsp/commands.rs | 2 - 3 files changed, 57 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index a1bcf6679e2a..dd05ad23f763 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 +use core::cell::Cell; + use kernel::{ device, devres::Devres, @@ -7,7 +9,7 @@ gpu::buddy::GpuBuddyParams, pci, prelude::*, - sizes::{SZ_1M, SZ_4K}, + sizes::SZ_4K, sync::Arc, // }; @@ -28,6 +30,13 @@ regs, }; +/// Parameters extracted from GSP boot for initializing memory subsystems. +#[derive(Clone, Copy)] +struct BootParams { + usable_vram_start: u64, + usable_vram_size: u64, +} + macro_rules! define_chipset { ({ $($variant:ident = $value:expr),* $(,)* }) => { @@ -270,6 +279,13 @@ pub(crate) fn new<'a>( devres_bar: Arc<Devres<Bar0>>, bar: &'a Bar0, ) -> impl PinInit<Self, Error> + 'a { + // Cell to share boot parameters between GSP boot and subsequent initializations. + // Contains usable VRAM region from FbLayout and BAR1 PDE base from GSP info. + let boot_params: Cell<BootParams> = Cell::new(BootParams { + usable_vram_start: 0, + usable_vram_size: 0, + }); + try_pin_init!(Self { spec: Spec::new(pdev.as_ref(), bar).inspect(|spec| { dev_info!(pdev.as_ref(),"NVIDIA ({})\n", spec); @@ -291,18 +307,42 @@ pub(crate) fn new<'a>( sec2_falcon: Falcon::new(pdev.as_ref(), spec.chipset)?, - // Create GPU memory manager owning memory management resources. - // This will be initialized with the usable VRAM region from GSP in a later - // patch. For now, we use a placeholder of 1MB. - mm: GpuMm::new(devres_bar.clone(), GpuBuddyParams { - base_offset_bytes: 0, - physical_memory_size_bytes: SZ_1M as u64, - chunk_size_bytes: SZ_4K as u64, - })?, - gsp <- Gsp::new(pdev), - gsp_static_info: { gsp.boot(pdev, bar, spec.chipset, gsp_falcon, sec2_falcon)?.0 }, + // Boot GSP and extract usable VRAM region for buddy allocator. + gsp_static_info: { + let (info, fb_layout) = gsp.boot(pdev, bar, spec.chipset, gsp_falcon, sec2_falcon)?; + + let usable_vram = fb_layout.usable_vram.as_ref().ok_or_else(|| { + dev_err!(pdev.as_ref(), "No usable FB regions found from GSP\n"); + ENODEV + })?; + + dev_info!( + pdev.as_ref(), + "Using FB region: {:#x}..{:#x}\n", + usable_vram.start, + usable_vram.end + ); + + boot_params.set(BootParams { + usable_vram_start: usable_vram.start, + usable_vram_size: usable_vram.end - usable_vram.start, + }); + + info + }, + + // Create GPU memory manager owning memory management resources. + // Uses the usable VRAM region from GSP for buddy allocator. + mm: { + let params = boot_params.get(); + GpuMm::new(devres_bar.clone(), GpuBuddyParams { + base_offset_bytes: params.usable_vram_start, + physical_memory_size_bytes: params.usable_vram_size, + chunk_size_bytes: SZ_4K as u64, + })? + }, bar: devres_bar, }) diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 75f949bc4864..a034e2e80a4b 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -150,7 +150,7 @@ pub(crate) fn boot( let gsp_fw = KBox::pin_init(GspFirmware::new(dev, chipset, FIRMWARE_VERSION), GFP_KERNEL)?; - let fb_layout = FbLayout::new(chipset, bar, &gsp_fw)?; + let mut fb_layout = FbLayout::new(chipset, bar, &gsp_fw)?; dev_dbg!(dev, "{:#x?}\n", fb_layout); Self::run_fwsec_frts(dev, gsp_falcon, bar, &bios, &fb_layout)?; @@ -252,6 +252,11 @@ pub(crate) fn boot( Err(e) => dev_warn!(pdev.as_ref(), "GPU name unavailable: {:?}\n", e), } + // Populate usable VRAM from GSP response. + if let Some((base, size)) = info.usable_fb_region() { + fb_layout.set_usable_vram(base, size); + } + Ok((info, fb_layout)) } } diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index d619cf294b9c..4a7eda512789 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -191,7 +191,6 @@ pub(crate) struct GetGspStaticInfoReply { gpu_name: [u8; 64], bar1_pde_base: u64, /// First usable FB region (base, size) for memory allocation. - #[expect(dead_code)] usable_fb_region: Option<(u64, u64)>, } @@ -242,7 +241,6 @@ pub(crate) fn bar1_pde_base(&self) -> u64 { /// Returns the usable FB region (base, size) for driver allocation which is /// already retrieved from the GSP. - #[expect(dead_code)] pub(crate) fn usable_fb_region(&self) -> Option<(u64, u64)> { self.usable_fb_region } -- 2.34.1
