From: Zhi Wang <[email protected]> Start an allocated instance's GSP plugin with its channel map, VRAM, plugin heap and log regions. Clear the old ready marker before BOOTLOAD and wait for firmware to publish a new one.
Retain the command queue and memory dependencies needed by the instance, and map its communication buffers before boot. Signed-off-by: Zhi Wang <[email protected]> Co-developed-by: Alok Kumar <[email protected]> Signed-off-by: Alok Kumar <[email protected]> --- drivers/gpu/nova-core/gpu.rs | 71 +++++--- drivers/gpu/nova-core/gsp/fw/commands.rs | 1 - drivers/gpu/nova-core/vgpu.rs | 30 +++- drivers/gpu/nova-core/vgpu/commands.rs | 21 ++- drivers/gpu/nova-core/vgpu/fw.rs | 3 + drivers/gpu/nova-core/vgpu/fw/commands.rs | 2 - drivers/gpu/nova-core/vgpu/gsp_plugin_comm.rs | 16 +- drivers/gpu/nova-core/vgpu/instance.rs | 152 ++++++++++++++++-- drivers/gpu/nova-core/vgpu/vram.rs | 1 - 9 files changed, 244 insertions(+), 53 deletions(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 04cdffc3de2b..da753bfc2df7 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -8,6 +8,7 @@ fmt, gpu::buddy::GpuBuddyParams, io::Io, + new_mutex, num::Bounded, pci, prelude::*, @@ -16,6 +17,7 @@ SizeConstants, SZ_4K, // }, + sync::Mutex, uapi, // }; @@ -402,6 +404,7 @@ unsafe fn new( #[pin_data] pub(crate) struct Gpu<'gpu> { pub(crate) spec: Spec, + /// Drops before the MM, BAR1 mappings and GSP needed for instance teardown. vgpu: Option<Pin<KBox<VgpuManager<'gpu>>>>, /// GSP event interrupt registration, and the enable of its subtree. /// @@ -414,7 +417,8 @@ pub(crate) struct Gpu<'gpu> { /// /// Must be kept declared *before* `gsp_resources`, so that its components are dropped while /// the GSP is still operational. - mm: GpuMm<'gpu>, + #[pin] + mm: Mutex<GpuMm<'gpu>>, /// BAR1 user interface for CPU access to GPU virtual memory. #[pin] bar_user: BarUser<'gpu>, @@ -540,23 +544,6 @@ pub(crate) fn new<'a>( })?, }), - vgpu: { - let info = &gsp_resources.boot_result.static_info; - match gsp_resources.vgpu_state { - VgpuState::Disabled => None, - VgpuState::Enabled { .. } => Some(KBox::pin_init(VgpuManager::new( - // SAFETY: `chid_pool` is initialized above at its final pinned address. - // The private manager and its pool borrow cannot escape this `Gpu`. - // Completed field drop order drops the manager before the pool; on failure, - // pin-init drops it before the earlier-initialized pool. - unsafe { &*core::ptr::from_ref(chid_pool.as_ref().get_ref()) }, - &info.fifo_engine_list(), - info.vmmu_segment_size, - TOTAL_CHANNELS, - ), GFP_KERNEL)?), - } - }, - _: { irq::gsp::quiesce(tree_ref, &gsp_resources.gsp_falcon); }, @@ -602,7 +589,7 @@ pub(crate) fn new<'a>( }, // Create GPU memory manager owning memory management resources. - mm: { + mm <- { let info = gsp_resources.static_info(); let usable_vram = info.usable_fb_regions.first().ok_or(ENODEV)?; let buddy_params = GpuBuddyParams { @@ -611,12 +598,15 @@ pub(crate) fn new<'a>( chunk_size: Alignment::new::<SZ_4K>(), }; - GpuMm::new( - bar, - gsp_resources.spec.chipset, - buddy_params, - VramAddress::from_raw(info.total_fb_end), - )? + new_mutex!( + GpuMm::new( + bar, + gsp_resources.spec.chipset, + buddy_params, + VramAddress::from_raw(info.total_fb_end), + )?, + "nova-core::gpu-mm", + ) }, // Create BAR1 user interface for CPU access to GPU virtual memory. @@ -632,6 +622,34 @@ pub(crate) fn new<'a>( bar1, ) }, + + vgpu: { + match gsp_resources.vgpu_state { + VgpuState::Disabled => None, + VgpuState::Enabled { .. } => { + // SAFETY: These sibling fields are initialized at their final pinned + // addresses. The private manager cannot escape this `Gpu`, and is dropped + // before all its dependencies, both here on failure and on normal removal. + let (cmdq, bar_user, mm, chid_pool) = unsafe { + ( + &*core::ptr::from_ref(&gsp_resources.gsp.cmdq), + &*core::ptr::from_ref(bar_user.as_ref().get_ref()), + &*core::ptr::from_ref(mm.as_ref().get_ref()), + &*core::ptr::from_ref(chid_pool.as_ref().get_ref()), + ) + }; + Some(KBox::pin_init(VgpuManager::new( + dev, + cmdq, + bar_user, + mm, + chid_pool, + gsp_resources.static_info(), + TOTAL_CHANNELS, + ), GFP_KERNEL)?) + } + } + }, }) } @@ -663,10 +681,11 @@ pub(crate) fn run_selftests(self: Pin<&mut Self>, pdev: &pci::Device<device::Bou let this = self.project(); let dev = pdev.as_ref(); let info = this.gsp_resources.static_info(); + let mut mm = this.mm.lock(); if let Err(err) = crate::mm::selftest::run( dev, - this.mm, + &mut mm, &info.usable_fb_regions, this.bar_user.as_ref().get_ref(), info.bar1_pde_base, diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs index bdc0764bb898..08798f150ad7 100644 --- a/drivers/gpu/nova-core/gsp/fw/commands.rs +++ b/drivers/gpu/nova-core/gsp/fw/commands.rs @@ -326,7 +326,6 @@ pub(crate) struct FifoEngineList { } impl FifoEngineList { - #[expect(dead_code)] pub(crate) fn gmc_ids(&self) -> &[u32] { // PANIC: The type invariant bounds `count` by the array capacity. &self.gmc_ids[..self.count] diff --git a/drivers/gpu/nova-core/vgpu.rs b/drivers/gpu/nova-core/vgpu.rs index 024cb13694f4..84c4d0b3839d 100644 --- a/drivers/gpu/nova-core/vgpu.rs +++ b/drivers/gpu/nova-core/vgpu.rs @@ -19,7 +19,17 @@ ChannelIdPool, Chipset, // }, - gsp::commands::FifoEngineList, // + gsp::{ + cmdq::Cmdq, + commands::{ + FifoEngineList, + GspStaticInfo, // + }, // + }, + mm::{ + bar_user::BarUser, + GpuMm, // + }, // }; mod commands; @@ -99,6 +109,10 @@ fn query_state( pub(crate) struct VgpuManager<'gpu> { #[pin] instances: Mutex<VgpuInstances<'gpu>>, + dev: &'gpu device::Device<device::Bound>, + cmdq: &'gpu Cmdq<'gpu>, + bar_user: &'gpu BarUser<'gpu>, + mm: &'gpu Mutex<GpuMm<'gpu>>, chid_pool: &'gpu ChannelIdPool, /// VMMU segment size in bytes, or zero if GSP-RM omitted it. vmmu_segment_size: u64, @@ -109,14 +123,22 @@ pub(crate) struct VgpuManager<'gpu> { impl<'gpu> VgpuManager<'gpu> { /// Retains runtime parameters from a completed vGPU-enabled GSP boot. pub(crate) fn new( + dev: &'gpu device::Device<device::Bound>, + cmdq: &'gpu Cmdq<'gpu>, + bar_user: &'gpu BarUser<'gpu>, + mm: &'gpu Mutex<GpuMm<'gpu>>, chid_pool: &'gpu ChannelIdPool, - fifo_engine_list: &FifoEngineList, - vmmu_segment_size: u64, + info: &GspStaticInfo, total_channels: u32, ) -> impl PinInit<Self> + use<'gpu> { - let fifo_engine_list = *fifo_engine_list; + let fifo_engine_list = info.fifo_engine_list(); + let vmmu_segment_size = info.vmmu_segment_size; pin_init!(Self { instances <- new_mutex!(VgpuInstances::new(), "nova-core::vgpu-instances"), + dev, + cmdq, + bar_user, + mm, chid_pool, vmmu_segment_size, total_channels, diff --git a/drivers/gpu/nova-core/vgpu/commands.rs b/drivers/gpu/nova-core/vgpu/commands.rs index dc2534175e11..6207211d7f9d 100644 --- a/drivers/gpu/nova-core/vgpu/commands.rs +++ b/drivers/gpu/nova-core/vgpu/commands.rs @@ -8,7 +8,9 @@ use kernel::{ device, - prelude::*, // + prelude::*, + time::Delta, + transmute::AsBytes, // }; use crate::gsp::{ @@ -26,6 +28,7 @@ commands::{ VgpuPropertiesSchema, // }, + GMCAPI_CMD_BOOTLOAD_GSP_VGPU_PLUGIN_TASK, GMCAPI_CMD_QUERY_ASSIGNED_VF_VGPU_TYPE, GMCAPI_CMD_QUERY_VGPU_PROPERTIES, // }, // @@ -90,3 +93,19 @@ pub(super) fn query_vgpu_properties( Ok(properties) } } + +/// Send BOOTLOAD and check its firmware status. +pub(super) fn send_bootload( + dev: &device::Device<device::Bound>, + cmdq: &Cmdq<'_>, + payload: &[u64], +) -> Result { + let command_id = GMCAPI_CMD_BOOTLOAD_GSP_VGPU_PLUGIN_TASK; + let response = cmdq.send_gmc_and_receive_timeout( + command_id, + AsBytes::as_bytes(payload), + 0, + Delta::from_secs(10), + )?; + check_status(dev, command_id, response.status) +} diff --git a/drivers/gpu/nova-core/vgpu/fw.rs b/drivers/gpu/nova-core/vgpu/fw.rs index aabe114b55c2..82c29dfb467a 100644 --- a/drivers/gpu/nova-core/vgpu/fw.rs +++ b/drivers/gpu/nova-core/vgpu/fw.rs @@ -30,3 +30,6 @@ pub(super) const GMCAPI_CMD_QUERY_VGPU_PROPERTIES: u32 = bindings::GMCAPI_COMMANDS_GMCAPI_CMD_QUERY_VGPU_PROPERTIES; + +pub(super) const GMCAPI_CMD_BOOTLOAD_GSP_VGPU_PLUGIN_TASK: u32 = + bindings::GMCAPI_COMMANDS_GMCAPI_CMD_BOOTLOAD_GSP_VGPU_PLUGIN_TASK; diff --git a/drivers/gpu/nova-core/vgpu/fw/commands.rs b/drivers/gpu/nova-core/vgpu/fw/commands.rs index b8007c3e4a71..3c4a6192d2a6 100644 --- a/drivers/gpu/nova-core/vgpu/fw/commands.rs +++ b/drivers/gpu/nova-core/vgpu/fw/commands.rs @@ -60,7 +60,6 @@ pub(crate) struct ChannelMapEntry(u64) { impl ChannelMapEntry { const KEY: KeyId = 0x1001; - #[expect(dead_code)] pub(crate) fn new(engine_type: u16, index: u16, chid_offset: u32) -> Self { Self::zeroed() .with_engine_type(engine_type) @@ -158,7 +157,6 @@ pub(crate) struct BootloadInfo<'a> { } /// Encodes a `VGPU_BOOTLOAD` request using the typed NVKV schema. -#[expect(dead_code)] pub(crate) fn encode_vgpu_bootload(info: BootloadInfo<'_>) -> Result<KVVec<u64>> { let request = VgpuBootloadRequest { dbdf: info.dbdf.into(), diff --git a/drivers/gpu/nova-core/vgpu/gsp_plugin_comm.rs b/drivers/gpu/nova-core/vgpu/gsp_plugin_comm.rs index b945e216a1d6..0b4d03c5f0fc 100644 --- a/drivers/gpu/nova-core/vgpu/gsp_plugin_comm.rs +++ b/drivers/gpu/nova-core/vgpu/gsp_plugin_comm.rs @@ -39,7 +39,6 @@ size_of::<RawControlRegion>() == u32_as_usize(fw::VGPU_CPU_GSP_CTRL_BUFF_REGION_SIZE) ); /// Physical VRAM regions containing the vGPU plugin logs. -#[expect(dead_code)] pub(super) struct PluginLogRegions { pub(super) init: VramRegion, pub(super) vgpu: VramRegion, @@ -86,7 +85,6 @@ pub(super) struct CommBufferRegion<'map, 'gpu> { kernel_log: VramRegion, } -#[expect(dead_code)] impl<'map, 'gpu> CommBufferRegion<'map, 'gpu> { /// Map the communication portion of a plugin management heap. pub(super) fn new( @@ -188,6 +186,19 @@ pub(super) fn plugin_logs(&self) -> PluginLogRegions { } } + /// Clear a previous boot marker before starting the plugin. + pub(super) fn clear_plugin_ready(&self) -> Result { + let offset = self.io_offset( + &self.control, + core::mem::offset_of!(RawControlRegion, __bindgen_anon_1.message_seq_num), + size_of::<u32>(), + )?; + self.map.try_write32(0, offset)?; + // Complete the posted clear before firmware can publish its new marker. + self.map.try_read32(offset)?; + Ok(()) + } + /// Return whether firmware has published the plugin boot marker. pub(super) fn is_plugin_ready(&self) -> Result<bool> { let value = self.read_u32( @@ -199,6 +210,7 @@ pub(super) fn is_plugin_ready(&self) -> Result<bool> { } /// Invalidate the PTEs and release the communication mapping. + #[expect(dead_code)] pub(super) fn unmap(&mut self) -> Result { self.map.unmap() } diff --git a/drivers/gpu/nova-core/vgpu/instance.rs b/drivers/gpu/nova-core/vgpu/instance.rs index a90aaef28b35..b2a0b25c0e15 100644 --- a/drivers/gpu/nova-core/vgpu/instance.rs +++ b/drivers/gpu/nova-core/vgpu/instance.rs @@ -7,18 +7,38 @@ }; use kernel::{ + device, prelude::*, ptr::Alignment, - sizes::SizeConstants, // + sizes::SizeConstants, + time::{ + delay::fsleep, + Delta, + Instant, + Monotonic, // + }, // }; use crate::{ gpu::ChannelIdReservation, - mm::GpuMm, // + gsp::{ + cmdq::Cmdq, + commands::FifoEngineList, // + }, + mm::GpuMm, }; use super::{ - commands::Dbdf, + commands::{ + send_bootload, + Dbdf, // + }, + fw::commands::{ + encode_vgpu_bootload, + BootloadInfo, + ChannelMapEntry, // + }, + gsp_plugin_comm::CommBufferRegion, vram::{ VgpuVramLayout, VgpuVramSlot, @@ -27,6 +47,52 @@ VgpuManager, // }; +/// Ready limit used by `vmiopd_negotiate_cpu_gsp_version()` for the same marker. +const PLUGIN_READY_TIMEOUT: Delta = Delta::from_secs(10); + +/// Per-engine channel budget reserved by the full SR-IOV plugin. +/// +/// The supported GB20x path keeps firmware's non-heavy default, which uses +/// `PLUGIN_ALLOCATED_CHANNELS_PER_ENGINE` rather than the heavy-mode budget. +const PLUGIN_CHANNELS_PER_ENGINE: u32 = 3; + +/// Build the typed channel mapping from the GSP FIFO engine list. +fn channel_mapping( + fifo_engine_list: &FifoEngineList, + chid_offset: u32, +) -> Result<KVVec<ChannelMapEntry>> { + let mut mapping = KVVec::new(); + for &gmc_id in fifo_engine_list.gmc_ids() { + // CAST: The mask leaves only the low 16 bits of the GMC engine ID. + let engine_type = (gmc_id & u32::from(u16::MAX)) as u16; + // CAST: Shifting a `u32` by 16 leaves at most 16 bits. + let index = (gmc_id >> u16::BITS) as u16; + mapping.push( + ChannelMapEntry::new(engine_type, index, chid_offset), + GFP_KERNEL, + )?; + } + Ok(mapping) +} + +fn wait_plugin_ready( + dev: &device::Device<device::Bound>, + comm: &CommBufferRegion<'_, '_>, +) -> Result { + let start = Instant::<Monotonic>::now(); + + loop { + if comm.is_plugin_ready()? { + dev_dbg!(dev, "vGPU plugin ready after {:?}\n", start.elapsed()); + return Ok(()); + } + if start.elapsed() >= PLUGIN_READY_TIMEOUT { + return Err(ETIMEDOUT); + } + fsleep(Delta::from_millis(1)); + } +} + /// Guest Function ID validated against one device's total number of VFs. /// /// GFID 0 is reserved for the PF; VFs start at 1. Keeping the nonzero `u16` @@ -49,7 +115,6 @@ pub(super) fn new(gfid: u32, total_vfs: NonZero<u16>) -> Result<Self> { } } - #[expect(dead_code)] pub(super) const fn get(self) -> u16 { self.0.get() } @@ -68,14 +133,72 @@ pub(super) struct VgpuType { } /// A vGPU instance and the resources reserved for it. -#[expect(dead_code)] struct VgpuInstance<'gpu> { gfid: Gfid, dbdf: Dbdf, vgpu_type: VgpuType, vm_pid: u32, - chids: ChannelIdReservation<'gpu>, + num_plugin_channels: u32, + comm: CommBufferRegion<'gpu, 'gpu>, + // Unmap the communication region before returning its slot and channel IDs. vram_slot: VgpuVramSlot, + chids: ChannelIdReservation<'gpu>, +} + +impl<'gpu> VgpuInstance<'gpu> { + #[expect(dead_code)] + fn activate(&mut self, vgpu: &VgpuManager<'gpu>) -> Result { + let dev = vgpu.dev; + self.bootload(dev, vgpu.cmdq, &vgpu.fifo_engine_list)?; + + Ok(()) + } + + /// Bootload the GSP vGPU plugin and wait for its BAR1 ready indication. + fn bootload( + &mut self, + dev: &device::Device<device::Bound>, + cmdq: &Cmdq<'_>, + fifo_engine_list: &FifoEngineList, + ) -> Result { + let fb = &self.vram_slot.fbmem; + let mgmt = &self.vram_slot.mgmt_heap; + let logs = self.comm.plugin_logs(); + + let payload = encode_vgpu_bootload(BootloadInfo { + dbdf: self.dbdf, + gfid: u32::from(self.gfid.get()), + vgpu_type: self.vgpu_type.vgpu_type_id, + vm_pid: self.vm_pid, + num_channels: u32::try_from(self.chids.len()).map_err(|_| EOVERFLOW)?, + num_plugin_channels: self.num_plugin_channels, + channel_mapping: channel_mapping( + fifo_engine_list, + u32::try_from(self.chids.start).map_err(|_| EOVERFLOW)?, + )?, + guest_fb: fb, + plugin_heap: mgmt, + ctrl_buffer_offset: 0, + init_log: &logs.init, + vgpu_log: &logs.vgpu, + kernel_log: &logs.kernel, + })?; + + dev_dbg!( + dev, + "bootload: gfid={} sending {} typed NVKV bytes\n", + self.gfid.get(), + payload.len() * size_of::<u64>(), + ); + + self.comm.clear_plugin_ready()?; + send_bootload(dev, cmdq, &payload)?; + + wait_plugin_ready(dev, &self.comm)?; + + dev_dbg!(dev, "bootload: gfid={} plugin ready\n", self.gfid.get()); + Ok(()) + } } /// Identity and firmware profile used to allocate an instance. @@ -133,12 +256,7 @@ fn alloc_vram_slot(&mut self, mm: &GpuMm<'_>, layout: VgpuVramLayout) -> Result< } /// Allocate resources and register a new inactive vGPU instance. - fn allocate_instance( - &mut self, - mm: &GpuMm<'_>, - vgpu: &VgpuManager<'gpu>, - info: InstanceInfo, - ) -> Result<Gfid> { + fn allocate_instance(&mut self, vgpu: &VgpuManager<'gpu>, info: InstanceInfo) -> Result<Gfid> { let InstanceInfo { gfid, dbdf, @@ -165,8 +283,7 @@ fn allocate_instance( return Err(ENOSPC); } - // Reserve registry capacity before acquiring resources so publishing - // the completed instance cannot fail due to memory pressure. + // Reserve capacity before acquiring resources so registration cannot allocate. self.instances.reserve(1, GFP_KERNEL)?; let channels_per_instance = vgpu @@ -187,15 +304,18 @@ fn allocate_instance( heap_size: vgpu_type.gsp_heap_size, fb_align: vgpu.vmmu_segment_size, }; - let vram_slot = self.alloc_vram_slot(mm, vram_layout)?; + let vram_slot = self.alloc_vram_slot(&vgpu.mm.lock(), vram_layout)?; + let comm = CommBufferRegion::new(vgpu.bar_user, vgpu.mm, &vram_slot.mgmt_heap)?; let instance = VgpuInstance { gfid, dbdf, vgpu_type, vm_pid, - chids, + num_plugin_channels: PLUGIN_CHANNELS_PER_ENGINE, + comm, vram_slot, + chids, }; self.instances .push_within_capacity(instance) diff --git a/drivers/gpu/nova-core/vgpu/vram.rs b/drivers/gpu/nova-core/vgpu/vram.rs index faaeea106009..bd7d1b4bf039 100644 --- a/drivers/gpu/nova-core/vgpu/vram.rs +++ b/drivers/gpu/nova-core/vgpu/vram.rs @@ -102,7 +102,6 @@ fn drop(&mut self) { /// All device accesses and mappings of its regions must end before dropping the slot. /// Clearing the entry locks a sleeping mutex, so dropping the slot may sleep. #[must_use] -#[expect(dead_code)] pub(super) struct VgpuVramSlot { pub(super) fbmem: VramRegion, pub(super) mgmt_heap: VramRegion, -- 2.53.0
