From: Joel Fernandes <[email protected]> Add the BAR1 user interface for CPU access to GPU virtual memory through the BAR1 aperture.
Signed-off-by: Joel Fernandes <[email protected]> [ecourtney: map BAR1 in NovaCore, borrow it in Gpu and BarUser, no Devres] [ecourtney: update for the VramAddress raw API and gsp_resources chipset] [ecourtney: drop the owned GpuMm, pass it mutably through map and release] Signed-off-by: Eliot Courtney <[email protected]> --- drivers/gpu/nova-core/driver.rs | 45 +++++++-- drivers/gpu/nova-core/gpu.rs | 32 ++++++- drivers/gpu/nova-core/gsp/commands.rs | 1 - drivers/gpu/nova-core/mm.rs | 1 + drivers/gpu/nova-core/mm/bar_user.rs | 171 ++++++++++++++++++++++++++++++++++ 5 files changed, 240 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 5723ff8f71ea..0672a0707a71 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -2,7 +2,11 @@ use kernel::{ auxiliary, - device::Core, + device::{ + Bound, + Core, // + }, + io::resource, pci, pci::{ Class, @@ -28,6 +32,7 @@ pub(crate) struct NovaCore<'bound> { #[pin] pub(crate) gpu: Gpu<'bound>, bar: pci::Bar<'bound, BAR0_SIZE>, + bar1: Bar1<'bound>, #[allow(clippy::type_complexity)] _reg: auxiliary::Registration<'bound, CovariantForLt!(())>, } @@ -38,9 +43,27 @@ pub(crate) struct NovaCore<'bound> { pub(crate) type Bar0<'a> = &'a pci::Bar<'a, BAR0_SIZE>; pub(crate) type NovaRegisters = kernel::io::Region<BAR0_SIZE>; -#[expect(dead_code)] pub(crate) type Bar1<'a> = pci::Bar<'a>; +/// Returns the Linux PCI resource index that holds BAR1 for an NVIDIA GPU. +/// +/// On Maxwell through Ada, BAR0 is a 32-bit memory BAR occupying a single +/// Linux PCI resource slot, so BAR1 lives at index 1. Starting with Blackwell +/// (and on some Ampere GA100 / Hopper SKUs) BAR0 is a 64-bit memory BAR that +/// consumes two consecutive resource slots: index 0 holds the low 32 bits and +/// index 1 holds the high 32 bits (with no `flags` / or size of its own), +/// shifting BAR1 to index 2. +pub(crate) fn bar1_resource_index(pdev: &pci::Device<Bound>) -> Result<u32> { + // Probe the `IORESOURCE_MEM_64` flag of BAR0 as a robust way of exposing + // if BAR0 and hence BAR1 is 64-bit. + let flags0 = pdev.resource_flags(0)?; + if flags0.contains(resource::Flags::IORESOURCE_MEM_64) { + Ok(2) + } else { + Ok(1) + } +} + kernel::pci_device_table!( PCI_TABLE, <NovaCoreDriver as pci::Driver>::IdInfo, @@ -82,12 +105,18 @@ fn probe<'bound>( Ok(try_pin_init!(NovaCore { bar: pdev.iomap_region_sized::<BAR0_SIZE>(0, c"nova-core/bar0")?, - // TODO: Use `&bar` self-referential pin-init syntax once available. - // - // SAFETY: `bar` is initialized before this expression is evaluated - // (`try_pin_init!()` initializes fields in declaration order), lives at a pinned - // stable address, and is dropped after `gpu` (struct field drop order). - gpu <- Gpu::new(pdev, unsafe { &*core::ptr::from_ref(bar) }), + bar1: { + let bar1_idx = bar1_resource_index(pdev)?; + pdev.iomap_region(bar1_idx, c"nova-core/bar1")? + }, + // TODO: Use self-referential pin-init syntax once available. + gpu <- Gpu::new( + pdev, + // SAFETY: `bar` is initialized above, pinned, and outlives `gpu`. + unsafe { &*core::ptr::from_ref(bar) }, + // SAFETY: `bar1` is initialized above, pinned, and outlives `gpu`. + unsafe { &*core::ptr::from_ref(bar1) }, + ), // Run optional GPU selftests. #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] _: { gpu.run_selftests(pdev) }, diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index b797472279d3..f72a92e045e5 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -16,11 +16,15 @@ SizeConstants, SZ_4K, // }, + sync::Arc, }; use crate::{ bounded_enum, - driver::Bar0, + driver::{ + Bar0, + Bar1, // + }, falcon::{ gsp::Gsp as GspFalcon, sec2::Sec2 as Sec2Falcon, @@ -35,6 +39,8 @@ GspBootContext, // }, mm::{ + bar_user::BarUser, + pagetable::MmuVersion, GpuMm, VramAddress, // }, @@ -148,6 +154,11 @@ pub(crate) const fn arch(self) -> Architecture { pub(crate) fn pci_config_mirror_range(self) -> Range<u32> { hal::gpu_hal(self).pci_config_mirror_range() } + + /// Returns the MMU version for this chipset. + pub(crate) fn mmu_version(self) -> MmuVersion { + MmuVersion::from(self.arch()) + } } // TODO @@ -297,6 +308,8 @@ pub(crate) struct Gpu<'gpu> { /// Must be kept declared *before* `gsp_resources`, so that its components are dropped while /// the GSP is still operational. mm: GpuMm<'gpu>, + /// BAR1 user interface for CPU access to GPU virtual memory. + bar_user: Arc<BarUser<'gpu>>, /// GSP and its resources. #[pin] gsp_resources: GspResources<'gpu>, @@ -340,6 +353,7 @@ impl<'gpu> Gpu<'gpu> { pub(crate) fn new<'a>( pdev: &'gpu pci::Device<device::Core<'a>>, bar: Bar0<'gpu>, + bar1: &'gpu Bar1<'gpu>, ) -> impl PinInit<Self, Error> + use<'gpu, 'a> { let dev = pdev.as_ref(); @@ -442,6 +456,22 @@ pub(crate) fn new<'a>( VramAddress::from_raw(gsp_static_info.total_fb_end), )? }, + + // Create BAR1 user interface for CPU access to GPU virtual memory. + bar_user: { + let pdb_addr = VramAddress::from_raw(gsp_static_info.bar1_pde_base); + let bar1_idx = crate::driver::bar1_resource_index(pdev)?; + let bar1_size = pdev.resource_len(bar1_idx)?; + Arc::pin_init( + BarUser::new( + pdb_addr, + gsp_resources.spec.chipset, + bar1_size, + bar1, + )?, + GFP_KERNEL, + )? + }, }) } diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index 1c467718c679..663e1d124781 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -215,7 +215,6 @@ fn init(&self) -> impl Init<Self::Command, Self::InitError> { pub(crate) struct GetGspStaticInfoReply { gpu_name: [u8; 64], /// BAR1 Page Directory Entry base address. - #[expect(dead_code)] pub(crate) bar1_pde_base: u64, /// Usable FB (VRAM) regions for driver memory allocation. pub(crate) usable_fb_regions: KVec<Range<u64>>, diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index 2cf37254fbb9..e04497256519 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -60,6 +60,7 @@ macro_rules! impl_pfn_bounded { pub(crate) use tlb::Tlb; +pub(crate) mod bar_user; mod hal; pub(super) mod pagetable; mod pramin; diff --git a/drivers/gpu/nova-core/mm/bar_user.rs b/drivers/gpu/nova-core/mm/bar_user.rs new file mode 100644 index 000000000000..ef1d8e6f8c9c --- /dev/null +++ b/drivers/gpu/nova-core/mm/bar_user.rs @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! BAR1 user interface for CPU access to GPU virtual memory. Used for USERD +//! for GPU work submission, and applications to access GPU buffers via mmap(). + +use kernel::{ + io::Io, + new_mutex, + prelude::*, + sync::{ + Arc, + Mutex, // + }, +}; + +use crate::{ + driver::Bar1, + gpu::Chipset, + mm::{ + vmm::{ + MappedRange, + Vmm, // + }, + GpuMm, + Pfn, + Vfn, + VirtualAddress, + VramAddress, + PAGE_SIZE, // + }, + num::IntoSafeCast, +}; + +/// BAR1 user interface for virtual memory mappings. +/// +/// Owns the [`Vmm`] for the BAR1 address space. +#[pin_data] +pub(crate) struct BarUser<'gpu> { + #[pin] + vmm: Mutex<Vmm>, + bar1: &'gpu Bar1<'gpu>, +} + +impl<'gpu> BarUser<'gpu> { + /// Create a pin-initializer for [`BarUser`]. + pub(crate) fn new( + pdb_addr: VramAddress, + chipset: Chipset, + va_size: u64, + bar1: &'gpu Bar1<'gpu>, + ) -> Result<impl PinInit<Self> + 'gpu> { + let vmm = Vmm::new(pdb_addr, chipset.mmu_version(), va_size)?; + Ok(pin_init!(Self { + vmm <- new_mutex!(vmm, "bar_user_vmm"), + bar1, + })) + } + + /// Map physical pages to a contiguous BAR1 virtual range. + pub(crate) fn map( + self: &Arc<Self>, + mm: &mut GpuMm<'_>, + pfns: &[Pfn], + writable: bool, + ) -> Result<BarUserAccess<'gpu>> { + if pfns.is_empty() { + return Err(EINVAL); + } + let mut vmm = self.vmm.lock(); + let mapped = vmm.map_pages(mm, pfns, None, writable)?; + + Ok(BarUserAccess { + bar_user: self.clone(), + mapped: Some(mapped), + }) + } +} + +/// Access object for a mapped BAR1 region. +pub(crate) struct BarUserAccess<'gpu> { + bar_user: Arc<BarUser<'gpu>>, + /// [`BarUserAccess::release`] [`Option::take`]s this; `Some` at + /// drop time means `release()` was never called. + mapped: Option<MappedRange>, +} + +impl BarUserAccess<'_> { + /// Tear down the BAR1 mapping. + pub(crate) fn release(mut self, mm: &mut GpuMm<'_>) -> Result { + let mapped = self.mapped.take().ok_or(EINVAL)?; + let mut vmm = self.bar_user.vmm.lock(); + vmm.unmap_pages(mm, mapped)?; + Ok(()) + } + + /// Returns the active mapping. + fn mapped(&self) -> &MappedRange { + // `mapped` is only `None` after `take()` in `release`; hence unwrap() + // cannot panic here. + self.mapped.as_ref().unwrap() + } + + /// Get the base virtual address of this mapping. + pub(crate) fn base(&self) -> VirtualAddress { + VirtualAddress::from(self.mapped().vfn_start) + } + + /// Get the total size of the mapped region in bytes. + pub(crate) fn size(&self) -> usize { + self.mapped().num_pages * PAGE_SIZE + } + + /// Get the starting virtual frame number. + pub(crate) fn vfn_start(&self) -> Vfn { + self.mapped().vfn_start + } + + /// Get the number of pages in this mapping. + pub(crate) fn num_pages(&self) -> usize { + self.mapped().num_pages + } + + /// Translate an offset within this mapping to a BAR1 aperture offset. + fn bar_offset(&self, offset: usize) -> Result<usize> { + if offset >= self.size() { + return Err(EINVAL); + } + + let base_vfn: usize = self.mapped().vfn_start.raw().into_safe_cast(); + let base = base_vfn.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?; + base.checked_add(offset).ok_or(EOVERFLOW) + } + + // Fallible accessors with runtime bounds checking. + + /// Read a 32-bit value at the given offset. + pub(crate) fn try_read32(&self, offset: usize) -> Result<u32> { + let off = self.bar_offset(offset)?; + self.bar_user.bar1.try_read32(off) + } + + /// Write a 32-bit value at the given offset. + pub(crate) fn try_write32(&self, value: u32, offset: usize) -> Result { + let off = self.bar_offset(offset)?; + self.bar_user.bar1.try_write32(value, off) + } + + /// Read a 64-bit value at the given offset. + pub(crate) fn try_read64(&self, offset: usize) -> Result<u64> { + let off = self.bar_offset(offset)?; + self.bar_user.bar1.try_read64(off) + } + + /// Write a 64-bit value at the given offset. + pub(crate) fn try_write64(&self, value: u64, offset: usize) -> Result { + let off = self.bar_offset(offset)?; + self.bar_user.bar1.try_write64(value, off) + } +} + +impl Drop for BarUserAccess<'_> { + fn drop(&mut self) { + if self.mapped.is_some() { + kernel::pr_warn!( + "BarUserAccess dropped without calling release(). BarUser address space will leak.\n" + ); + } + // The inner `MappedRange`'s own `MustUnmapGuard` will also fire, + // identifying the leaked VA range. + } +} -- 2.55.0
