--- Begin Message ---
Package: qemu-system-x86
Version: 1:5.2+dfsg-9
When I try to boot an AMD SEV enabled guest OS it crashes immediately after the
"Loading initial ramdisk ..." message (the result is infinite reboot loop).
Here is my libvirt XML which works fine without the
"<launchSecurity...>...<launchSecurity/>" XML element (which enables the AMD
SEV):
<domain type='kvm'>
<name>test</name>
<uuid>4dea22b3-1d52-d8f3-2516-782e98ab3fa0</uuid>
<genid>ab826c0a-a349-4470-bbb9-d2b6ee23e8c3</genid>
<title>Test Server</title>
<memory dumpCore='off' unit='KiB'>8388608</memory>
<currentMemory unit='KiB'>8388608</currentMemory>
<memtune>
<hard_limit unit='KiB'>9437184</hard_limit>
</memtune>
<memoryBacking>
<hugepages>
<page size='2048' unit='KiB'/>
</hugepages>
<locked/>
<source type='memfd'/>
<access mode='shared'/>
<allocation mode='immediate'/>
<discard/>
</memoryBacking>
<vcpu placement='static'>4</vcpu>
<iothreads>1</iothreads>
<os firmware='efi'>
<type arch='x86_64' machine='pc-q35-5.2'>hvm</type>
<loader secure='yes'/>
<nvram>/var/lib/libvirt/qemu/nvram/test_VARS.fd</nvram>
<bootmenu enable='yes' timeout='5000'/>
<bios useserial='yes'/>
<smbios mode='emulate'/>
</os>
<features>
<acpi/>
<apic/>
<pae/>
<hap state='on'/>
<privnet/>
<kvm>
<hidden state='off'/>
<hint-dedicated state='on'/>
<poll-control state='on'/>
</kvm>
<pvspinlock state='on'/>
<ioapic driver='kvm'/>
</features>
<cpu mode='host-passthrough' check='none' migratable='off'>
<topology sockets='1' dies='1' cores='2' threads='2'/>
<cache mode='passthrough'/>
<feature policy='require' name='topoext'/>
</cpu>
<clock offset='utc'>
<timer name='kvmclock' present='yes'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>coredump-restart</on_crash>
<on_lockfailure>restart</on_lockfailure>
<devices>
<emulator>/usr/bin/kvm</emulator>
<disk type='file' device='cdrom'>
<driver name='qemu' type='raw'/>
<source file='/home/data/debian.iso'/>
<target dev='sda' bus='sata'/>
<readonly/>
<boot order='2'/>
<address type='drive' controller='0' bus='0' target='0' unit='0'/>
</disk>
<disk type='block' device='disk' model='virtio-non-transitional'>
<driver name='qemu' type='raw' cache='none' io='native' iothread='1'
iommu='on'/>
<source dev='/dev/vg_storage/lv_test-swap'/>
<target dev='vda' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x03' slot='0x00'
function='0x0'/>
</disk>
<disk type='block' device='disk' model='virtio-non-transitional'>
<driver name='qemu' type='raw' cache='none' io='native' iothread='1'
iommu='on'/>
<source dev='/dev/vg_storage/lv_test-root'/>
<target dev='vdb' bus='virtio'/>
<boot order='1'/>
<address type='pci' domain='0x0000' bus='0x04' slot='0x00'
function='0x0'/>
</disk>
<controller type='usb' index='0' model='qemu-xhci'>
<driver iommu='on'/>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00'
function='0x0'/>
</controller>
<controller type='sata' index='0'>
<driver iommu='on'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1f'
function='0x2'/>
</controller>
<controller type='pci' index='0' model='pcie-root'>
<driver iommu='on'/>
</controller>
<controller type='pci' index='1' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='1' port='0x10'/>
<driver iommu='on'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'
multifunction='on'/>
</controller>
<controller type='pci' index='2' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='2' port='0x11'/>
<driver iommu='on'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01'
function='0x1'/>
</controller>
<controller type='pci' index='3' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='3' port='0x12'/>
<driver iommu='on'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01'
function='0x2'/>
</controller>
<controller type='pci' index='4' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='4' port='0x13'/>
<driver iommu='on'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01'
function='0x3'/>
</controller>
<controller type='pci' index='5' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='5' port='0x14'/>
<driver iommu='on'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01'
function='0x4'/>
</controller>
<controller type='pci' index='6' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='6' port='0x15'/>
<driver iommu='on'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01'
function='0x5'/>
</controller>
<controller type='pci' index='7' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='7' port='0x16'/>
<driver iommu='on'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01'
function='0x6'/>
</controller>
<controller type='virtio-serial' index='0'>
<driver iommu='on'/>
<address type='pci' domain='0x0000' bus='0x06' slot='0x00'
function='0x0'/>
</controller>
<interface type='bridge' trustGuestRxFilters='no'>
<mac address='52:54:00:44:48:04' type='static'/>
<source bridge='wanbr0'/>
<model type='virtio'/>
<driver iommu='on'/>
<filterref filter='clean-traffic'>
<parameter name='IP' value='192.168.1.101'/>
</filterref>
<link state='up'/>
<rom enabled='no'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x00'
function='0x0'/>
</interface>
<serial type='pty'>
<target type='isa-serial' port='0'>
<model name='isa-serial'/>
</target>
</serial>
<console type='pty'>
<target type='serial' port='0'/>
</console>
<channel type='unix'>
<target type='virtio' name='org.qemu.guest_agent.0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
<input type='mouse' bus='ps2'>
<driver iommu='on'/>
</input>
<input type='keyboard' bus='ps2'>
<driver iommu='on'/>
</input>
<graphics type='vnc' port='5901' autoport='no' listen='127.0.0.1'
sharePolicy='allow-exclusive'>
<listen type='address' address='127.0.0.1'/>
</graphics>
<video>
<model type='bochs' vram='16384' heads='1' primary='yes'/>
<address type='pci' domain='0x0000' bus='0x05' slot='0x00'
function='0x0'/>
</video>
<memballoon model='none'/>
<rng model='virtio'>
<backend model='random'>/dev/random</backend>
<driver iommu='on'/>
<address type='pci' domain='0x0000' bus='0x07' slot='0x00'
function='0x0'/>
</rng>
</devices>
<launchSecurity type='sev'>
<cbitpos>47</cbitpos>
<reducedPhysBits>1</reducedPhysBits>
<policy>0x0033</policy>
</launchSecurity>
</domain>
kvm command line arguments (from the XML above):
/usr/bin/kvm -name guest=test,debug-threads=on -S -object
secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-12-test/master-key.aes
-blockdev
{"driver":"file","filename":"/usr/share/OVMF/OVMF_CODE_4M.ms.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"}
-blockdev
{"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"}
-blockdev
{"driver":"file","filename":"/var/lib/libvirt/qemu/nvram/test_VARS.fd","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap"}
-blockdev
{"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"}
-machine
pc-q35-5.2,accel=kvm,usb=off,smm=on,dump-guest-core=off,kernel_irqchip=on,memory-encryption=sev0,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,memory-backend=pc.ram
-cpu
host,migratable=off,topoext=on,kvmclock=on,kvm-pv-unhalt=on,kvm-hint-dedicated=on,kvm-poll-control=on,host-cache-info=on,l3-cache=off
-global driver=cfi.pflash01,property=secure,value=on -m 8192 -object
memory-backend-memfd,id=pc.ram,hugetlb=yes,hugetlbsize=2097152,share=yes,prealloc=yes,size=8589934592
-overcommit mem-lock=on -smp 4,sockets=1,dies=1,cores=2,threads=2 -object
iothread,id=iothread1 -uuid 4dea22b3-1d52-d8f3-2516-782e98ab3fa0 -device
vmgenid,guid=ab826c0a-a349-4470-bbb9-d2b6ee23e8c3,id=vmgenid0 -no-user-config
-nodefaults -device sga -chardev socket,id=charmonitor,fd=33,server,nowait -mon
chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown -boot
menu=on,splash-time=5000,strict=on -device
pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x1
-device pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x1.0x1
-device pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x1.0x2
-device pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x1.0x3
-device pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x1.0x4
-device pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x1.0x5
-device pcie-root-port,port=0x16,chassis=7,id=pci.7,bus=pcie.0,addr=0x1.0x6
-device qemu-xhci,id=usb,bus=pci.2,addr=0x0 -device
virtio-serial-pci,id=virtio-serial0,iommu_platform=on,bus=pci.6,addr=0x0
-blockdev
{"driver":"file","filename":"/home/data/debian.iso","node-name":"libvirt-3-storage","auto-read-only":true,"discard":"unmap"}
-blockdev
{"node-name":"libvirt-3-format","read-only":true,"driver":"raw","file":"libvirt-3-storage"}
-device ide-cd,bus=ide.0,drive=libvirt-3-format,id=sata0-0-0,bootindex=2
-blockdev
{"driver":"host_device","filename":"/dev/vg_storage/lv_test-swap","aio":"native","node-name":"libvirt-2-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"}
-blockdev
{"node-name":"libvirt-2-format","read-only":false,"cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-2-storage"}
-device
virtio-blk-pci-non-transitional,iothread=iothread1,iommu_platform=on,bus=pci.3,addr=0x0,drive=libvirt-2-format,id=virtio-disk0,write-cache=on
-blockdev
{"driver":"host_device","filename":"/dev/vg_storage/lv_test-root","aio":"native","node-name":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"}
-blockdev
{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"}
-device
virtio-blk-pci-non-transitional,iothread=iothread1,iommu_platform=on,bus=pci.4,addr=0x0,drive=libvirt-1-format,id=virtio-disk1,bootindex=1,write-cache=on
-netdev tap,fd=35,id=hostnet0,vhost=on,vhostfd=36 -device
virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:44:48:04,bus=pci.1,addr=0x0,romfile=,iommu_platform=on
-chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0
-chardev socket,id=charchannel0,fd=37,server,nowait -device
virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0
-vnc 127.0.0.1:1,share=allow-exclusive -device
bochs-display,id=video0,vgamem=16384k,bus=pci.5,addr=0x0 -object
rng-random,id=objrng0,filename=/dev/random -device
virtio-rng-pci,rng=objrng0,id=rng0,iommu_platform=on,bus=pci.7,addr=0x0 -object
sev-guest,id=sev0,cbitpos=47,reduced-phys-bits=1,policy=0x33 -sandbox
on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -msg
timestamp=on
The configuration of the host OS related to AMD SEV (hardware: HP Proliant
DL325 Gen10 with fully updated BIOS/firmware):
BIOS/Platform Configuration (RBSU)
Memory Options -> Transparent Secure Memory Encryption: Enabled
Memory Options -> AMD Secure Memory Encryption: Enabled
lscpu
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Byte Order: Little Endian
Address sizes: 44 bits physical, 48 bits virtual
CPU(s): 48
On-line CPU(s) list: 0-47
Thread(s) per core: 2
Core(s) per socket: 24
Socket(s): 1
NUMA node(s): 1
Vendor ID: AuthenticAMD
CPU family: 23
Model: 49
Model name: AMD EPYC 7402P 24-Core Processor
Stepping: 0
Frequency boost: enabled
CPU MHz: 1495.405
CPU max MHz: 3349.6089
CPU min MHz: 1500.0000
BogoMIPS: 5589.65
Virtualization: AMD-V
L1d cache: 768 KiB
L1i cache: 768 KiB
L2 cache: 12 MiB
L3 cache: 128 MiB
NUMA node0 CPU(s): 0-47
Vulnerability Itlb multihit: Not affected
Vulnerability L1tf: Not affected
Vulnerability Mds: Not affected
Vulnerability Meltdown: Not affected
Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled
via prctl and seccomp
Vulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and
__user pointer sanitization
Vulnerability Spectre v2: Mitigation; Full AMD retpoline, IBPB
conditional, IBRS_FW, STIBP conditional, RSB filling
Vulnerability Srbds: Not affected
Vulnerability Tsx async abort: Not affected
Flags: fpu vme de pse tsc msr pae mce cx8 apic sep
mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext
fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid
extd_apicid aperfmperf pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2
x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic
cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext
perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate
sme ssbd mba sev ibrs ibpb stibp vmmcall sev_es fsgsbase bmi1 avx2 smep bmi2
cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves
cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr
rdpru wbnoinvd amd_ppin arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean
flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif
umip rdpid overflow_recov succor smca
/etc/default/grub
GRUB_CMDLINE_LINUX="mem_encrypt=on kvm_amd.sev=1"
dmesg | grep -i sev
[ 0.000000] Command line: BOOT_IMAGE=/vmlinuz-5.10.0-5-amd64
root=/dev/mapper/vg_storage-lv_host--root ro mem_encrypt=on kvm_amd.sev=1 quiet
[ 0.018303] Kernel command line: BOOT_IMAGE=/vmlinuz-5.10.0-5-amd64
root=/dev/mapper/vg_storage-lv_host--root ro mem_encrypt=on kvm_amd.sev=1 quiet
[ 4.350647] ccp 0000:42:00.1: sev enabled
[ 4.357947] ccp 0000:42:00.1: firmware: failed to load
amd/amd_sev_fam17h_model31h.sbin (-2)
[ 4.358816] ccp 0000:42:00.1: firmware: failed to load
amd/amd_sev_fam17h_model3xh.sbin (-2)
[ 4.359214] ccp 0000:42:00.1: firmware: failed to load amd/sev.fw (-2)
[ 4.381159] ccp 0000:42:00.1: SEV API:0.24 build:6
[ 4.424216] SEV supported
cat /proc/cmdline
BOOT_IMAGE=/vmlinuz-5.10.0-5-amd64 root=/dev/mapper/vg_storage-lv_host--root ro
mem_encrypt=on kvm_amd.sev=1 quiet
cat /sys/module/kvm_amd/parameters/sev
1
virsh domcapabilities | grep -i sev
<sev supported='yes'></sev>
/etc/apparmor.d/local/usr.sbin.libvirtd
capability bpf,
capability perfmon,
/etc/apparmor.d/local/abstractions/libvirt-qemu
/ w,
/usr/share/OVMF/* k,
/dev/sev rw,
... but the result is the same even if I uninstall/disable apparmor (instead of
the local configuration changes above).
The configuration of the guest OS (related to AMD SEV):
/etc/default/grub
GRUB_CMDLINE_LINUX="console=ttyS0,115200 mem_encrypt=on kvm_amd.sev=1"
but I also tried the following combinations (the result is always the same)
GRUB_CMDLINE_LINUX="console=ttyS0,115200 kvm_amd.sev=1"
GRUB_CMDLINE_LINUX="console=ttyS0,115200"
Summary: even if I follow all the kvm online manuals, the guest crashes
immediately after showing the "Loading initial ramdisk ..." (the result is
infinite reboot loop). There is NO error/warning in dmesg, /var/log/syslog,
/var/log/libvirt/qemu/test.log, serial console, ...
Host OS and guest OS information:
Debian version: bullseye/sid (fully updated)
Kernel: linux-image-5.10.0-5-amd64 (5.10.26-1)
GNU C Library: libc-bin (2.31-11)
--- End Message ---
