The registry object "RMSetSriovMode" is required to be set when vGPU is enabled.
Set "RMSetSriovMode" to 1 when nova-core is loading the GSP firmware and initialize the GSP registry objects, if vGPU is enabled. Cc: Timur Tabi <[email protected]> Cc: Joel Fernandes <[email protected]> Cc: Alexandre Courbot <[email protected]> Signed-off-by: Zhi Wang <[email protected]> --- drivers/gpu/nova-core/gsp/boot.rs | 4 +- drivers/gpu/nova-core/gsp/commands.rs | 89 +++++++++++++++++---------- drivers/gpu/nova-core/gsp/fw.rs | 44 +++++++++++++ 3 files changed, 104 insertions(+), 33 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 921d5e892f8a..ed8729041d46 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -364,7 +364,7 @@ pub(crate) fn boot( self.cmdq .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset, vf_info))?; self.cmdq - .send_command_no_wait(bar, commands::SetRegistry::new())?; + .send_command_no_wait(bar, commands::SetRegistry::new(ctx.vgpu_requested)?)?; Self::boot_via_sec2( dev, @@ -406,7 +406,7 @@ pub(crate) fn boot( self.cmdq .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset, vf_info))?; self.cmdq - .send_command_no_wait(bar, commands::SetRegistry::new())?; + .send_command_no_wait(bar, commands::SetRegistry::new(ctx.vgpu_requested)?)?; } // SEC2-based architectures need to run the GSP sequencer diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index 0445d05990e7..c96580d0f433 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -75,38 +75,62 @@ struct RegistryEntry { } /// The `SetRegistry` command. +/// +/// Registry entries are built dynamically at runtime based on the current +/// configuration (e.g. whether vGPU is enabled). pub(crate) struct SetRegistry { - entries: [RegistryEntry; Self::NUM_ENTRIES], + entries: KVec<RegistryEntry>, } impl SetRegistry { - // For now we hard-code the registry entries. Future work will allow others to - // be added as module parameters. - const NUM_ENTRIES: usize = 3; - - /// Creates a new `SetRegistry` command, using a set of hardcoded entries. - pub(crate) fn new() -> Self { - Self { - entries: [ - // RMSecBusResetEnable - enables PCI secondary bus reset - RegistryEntry { - key: "RMSecBusResetEnable", - value: 1, - }, - // RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration registers on - // any PCI reset. - RegistryEntry { - key: "RMForcePcieConfigSave", - value: 1, - }, - // RMDevidCheckIgnore - allows GSP-RM to boot even if the PCI dev ID is not found - // in the internal product name database. + /// Creates a new `SetRegistry` command. + /// + /// The base set of registry entries is always included. Additional entries + /// are appended dynamically based on runtime conditions (e.g. vGPU). + pub(crate) fn new(vgpu_requested: bool) -> Result<Self> { + let mut entries = KVec::new(); + + // RMSecBusResetEnable - enables PCI secondary bus reset + entries.push( + RegistryEntry { + key: "RMSecBusResetEnable", + value: 1, + }, + GFP_KERNEL, + )?; + + // RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration registers on + // any PCI reset. + entries.push( + RegistryEntry { + key: "RMForcePcieConfigSave", + value: 1, + }, + GFP_KERNEL, + )?; + + // RMDevidCheckIgnore - allows GSP-RM to boot even if the PCI dev ID is not found + // in the internal product name database. + entries.push( + RegistryEntry { + key: "RMDevidCheckIgnore", + value: 1, + }, + GFP_KERNEL, + )?; + + // RMSetSriovMode - required when vGPU is enabled. + if vgpu_requested { + entries.push( RegistryEntry { - key: "RMDevidCheckIgnore", + key: "RMSetSriovMode", value: 1, }, - ], + GFP_KERNEL, + )?; } + + Ok(Self { entries }) } } @@ -117,28 +141,31 @@ impl CommandToGsp for SetRegistry { type InitError = Infallible; fn init(&self) -> impl Init<Self::Command, Self::InitError> { - PackedRegistryTable::init(Self::NUM_ENTRIES as u32, self.variable_payload_len() as u32) + PackedRegistryTable::init( + self.entries.len() as u32, + self.variable_payload_len() as u32, + ) } fn variable_payload_len(&self) -> usize { let mut key_size = 0; - for i in 0..Self::NUM_ENTRIES { - key_size += self.entries[i].key.len() + 1; // +1 for NULL terminator + for entry in self.entries.iter() { + key_size += entry.key.len() + 1; // +1 for NULL terminator } - Self::NUM_ENTRIES * size_of::<PackedRegistryEntry>() + key_size + self.entries.len() * size_of::<PackedRegistryEntry>() + key_size } fn init_variable_payload( &self, dst: &mut SBufferIter<core::array::IntoIter<&mut [u8], 2>>, ) -> Result { - let string_data_start_offset = - size_of::<PackedRegistryTable>() + Self::NUM_ENTRIES * size_of::<PackedRegistryEntry>(); + let string_data_start_offset = size_of::<PackedRegistryTable>() + + self.entries.len() * size_of::<PackedRegistryEntry>(); // Array for string data. let mut string_data = KVec::new(); - for entry in self.entries.iter().take(Self::NUM_ENTRIES) { + for entry in self.entries.iter() { dst.write_all( PackedRegistryEntry::new( (string_data_start_offset + string_data.len()) as u32, diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 6d56b9b920fb..ca01ac3af9c6 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -664,9 +664,50 @@ pub(crate) enum MsgFunction { OsErrorLog = bindings::NV_VGPU_MSG_EVENT_OS_ERROR_LOG, PostEvent = bindings::NV_VGPU_MSG_EVENT_POST_EVENT, RcTriggered = bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED, + GpuacctPerfmonUtilSamples = bindings::NV_VGPU_MSG_EVENT_GPUACCT_PERFMON_UTIL_SAMPLES, UcodeLibOsPrint = bindings::NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT, } +impl fmt::Display for MsgFunction { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + // Common function codes + MsgFunction::Nop => write!(f, "NOP"), + MsgFunction::SetGuestSystemInfo => write!(f, "SET_GUEST_SYSTEM_INFO"), + MsgFunction::AllocRoot => write!(f, "ALLOC_ROOT"), + MsgFunction::AllocDevice => write!(f, "ALLOC_DEVICE"), + MsgFunction::AllocMemory => write!(f, "ALLOC_MEMORY"), + MsgFunction::AllocCtxDma => write!(f, "ALLOC_CTX_DMA"), + MsgFunction::AllocChannelDma => write!(f, "ALLOC_CHANNEL_DMA"), + MsgFunction::MapMemory => write!(f, "MAP_MEMORY"), + MsgFunction::BindCtxDma => write!(f, "BIND_CTX_DMA"), + MsgFunction::AllocObject => write!(f, "ALLOC_OBJECT"), + MsgFunction::Free => write!(f, "FREE"), + MsgFunction::Log => write!(f, "LOG"), + MsgFunction::GetGspStaticInfo => write!(f, "GET_GSP_STATIC_INFO"), + MsgFunction::SetRegistry => write!(f, "SET_REGISTRY"), + MsgFunction::GspSetSystemInfo => write!(f, "GSP_SET_SYSTEM_INFO"), + MsgFunction::GspInitPostObjGpu => write!(f, "GSP_INIT_POST_OBJGPU"), + MsgFunction::GspRmControl => write!(f, "GSP_RM_CONTROL"), + MsgFunction::GetStaticInfo => write!(f, "GET_STATIC_INFO"), + + // Event codes + MsgFunction::GpuacctPerfmonUtilSamples => write!(f, "GPUACCT_PERFMON_UTIL_SAMPLES"), + MsgFunction::GspInitDone => write!(f, "INIT_DONE"), + MsgFunction::GspRunCpuSequencer => write!(f, "RUN_CPU_SEQUENCER"), + MsgFunction::PostEvent => write!(f, "POST_EVENT"), + MsgFunction::RcTriggered => write!(f, "RC_TRIGGERED"), + MsgFunction::MmuFaultQueued => write!(f, "MMU_FAULT_QUEUED"), + MsgFunction::OsErrorLog => write!(f, "OS_ERROR_LOG"), + MsgFunction::GspPostNoCat => write!(f, "NOCAT"), + MsgFunction::GspLockdownNotice => write!(f, "LOCKDOWN_NOTICE"), + MsgFunction::ContinuationRecord => write!(f, "CONTINUATION_RECORD"), + MsgFunction::UcodeLibOsPrint => write!(f, "LIBOS_PRINT"), + } + } +} + + impl TryFrom<u32> for MsgFunction { type Error = kernel::error::Error; @@ -709,6 +750,9 @@ fn try_from(value: u32) -> Result<MsgFunction> { bindings::NV_VGPU_MSG_EVENT_OS_ERROR_LOG => Ok(MsgFunction::OsErrorLog), bindings::NV_VGPU_MSG_EVENT_POST_EVENT => Ok(MsgFunction::PostEvent), bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED => Ok(MsgFunction::RcTriggered), + bindings::NV_VGPU_MSG_EVENT_GPUACCT_PERFMON_UTIL_SAMPLES => { + Ok(MsgFunction::GpuacctPerfmonUtilSamples) + } bindings::NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT => Ok(MsgFunction::UcodeLibOsPrint), _ => Err(EINVAL), } -- 2.51.0
