Hello
Few days ago I had an issue with getting PCIe hotplug working on AArch64
machine. Enabled PCI hotplug in kernel and then got hit by some issues.
Out setup is a bunch of aarch64 servers and we use OpenStack to provide
access to arm64 systems. OpenStack uses libvirt to control VMs and
allows to add network interfaces and disk volumes to running systems.
By libvirt AArch64 is treated as PCIe machine without legacy PCI slots.
So to hotplug anything you first need to have enough pcie-root-port
entries as it is described in Qemu docs/pcie.txt and by patch to libvirt
documentation [1][2].
1. https://bugs.linaro.org/attachment.cgi?id=782
2. https://www.redhat.com/archives/libvir-list/2017-July/msg01033.html
But things get complicated once you are going to 32 PCIe devices limit
(which in our setup will rather not happen). UEFI first takes ages to
boot just to land in UEFI shell as it forgot all PCIe devices. With 31
devices it boots (also after long time).
I attached two xml files with VM definitions (for use with virsh). Also
attached their qemu command lines. One has 31 PCI devices, second has
32. Both use [30] as rootfs.
30.
https://builds.96boards.org/snapshots/reference-platform/components/developer-cloud/debian/cloud-image/30/debian-cloud-image.qcow2
Output from 31 PCI devices VM:
root@unassigned-hostname:~# lspci;lspci |wc -l
00:00.0 Host bridge: Red Hat, Inc. Device 0008
00:01.0 PCI bridge: Red Hat, Inc. Device 000c
00:01.1 PCI bridge: Red Hat, Inc. Device 000c
00:01.2 PCI bridge: Red Hat, Inc. Device 000c
00:01.3 PCI bridge: Red Hat, Inc. Device 000c
00:01.4 PCI bridge: Red Hat, Inc. Device 000c
00:01.5 PCI bridge: Red Hat, Inc. Device 000c
00:01.6 PCI bridge: Red Hat, Inc. Device 000c
00:01.7 PCI bridge: Red Hat, Inc. Device 000c
00:02.0 PCI bridge: Red Hat, Inc. Device 000c
00:02.1 PCI bridge: Red Hat, Inc. Device 000c
00:02.2 PCI bridge: Red Hat, Inc. Device 000c
00:02.3 PCI bridge: Red Hat, Inc. Device 000c
00:02.4 PCI bridge: Red Hat, Inc. Device 000c
00:02.5 PCI bridge: Red Hat, Inc. Device 000c
00:02.6 PCI bridge: Red Hat, Inc. Device 000c
01:00.0 Ethernet controller: Red Hat, Inc Device 1041 (rev 01)
02:00.0 SCSI storage controller: Red Hat, Inc Device 1048 (rev 01)
03:00.0 Communication controller: Red Hat, Inc Device 1043 (rev 01)
04:00.0 SCSI storage controller: Red Hat, Inc Device 1042 (rev 01)
05:00.0 Ethernet controller: Red Hat, Inc Device 1041 (rev 01)
06:00.0 Ethernet controller: Red Hat, Inc Device 1041 (rev 01)
07:00.0 Ethernet controller: Red Hat, Inc Device 1041 (rev 01)
08:00.0 Ethernet controller: Red Hat, Inc Device 1041 (rev 01)
09:00.0 Ethernet controller: Red Hat, Inc Device 1041 (rev 01)
0a:00.0 Ethernet controller: Red Hat, Inc Device 1041 (rev 01)
0b:00.0 Ethernet controller: Red Hat, Inc Device 1041 (rev 01)
0c:00.0 Ethernet controller: Red Hat, Inc Device 1041 (rev 01)
0d:00.0 Ethernet controller: Red Hat, Inc Device 1041 (rev 01)
0e:00.0 Ethernet controller: Red Hat, Inc Device 1041 (rev 01)
0f:00.0 Ethernet controller: Red Hat, Inc Device 1041 (rev 01)
31
root@unassigned-hostname:~# lspci -t
-[0000:00]-+-00.0
+-01.0-[01]----00.0
+-01.1-[02]----00.0
+-01.2-[03]----00.0
+-01.3-[04]----00.0
+-01.4-[05]----00.0
+-01.5-[06]----00.0
+-01.6-[07]----00.0
+-01.7-[08]----00.0
+-02.0-[09]----00.0
+-02.1-[0a]----00.0
+-02.2-[0b]----00.0
+-02.3-[0c]----00.0
+-02.4-[0d]----00.0
+-02.5-[0e]----00.0
\-02.6-[0f]----00.0
>From what I was told some parts of that issue lies in UEFI, some in
Qemu, some in Linux kernel.
<domain type='kvm'>
<name>pci</name>
<uuid>5361dd27-bdc1-4178-b26f-323b4009b226</uuid>
<memory unit='KiB'>12312576</memory>
<currentMemory unit='KiB'>12312576</currentMemory>
<vcpu placement='static'>5</vcpu>
<os>
<type arch='aarch64' machine='virt-2.9'>hvm</type>
<loader readonly='yes' type='pflash'>/usr/share/AAVMF/AAVMF_CODE.fd</loader>
<nvram>/var/lib/libvirt/qemu/nvram/pci_VARS.fd</nvram>
<boot dev='hd'/>
</os>
<features>
<acpi/>
<gic version='2'/>
</features>
<cpu mode='host-passthrough' check='none'/>
<clock offset='utc'/>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<devices>
<emulator>/usr/bin/qemu-system-aarch64</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/home/linaro/virt/debian-30.qcow2'/>
<target dev='vda' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
</disk>
<controller type='scsi' index='0' model='virtio-scsi'>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
</controller>
<controller type='pci' index='0' model='pcie-root'/>
<controller type='pci' index='1' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='1' port='0x8'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='2' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='2' port='0x9'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
</controller>
<controller type='pci' index='3' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='3' port='0xa'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
</controller>
<controller type='pci' index='4' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='4' port='0xb'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x3'/>
</controller>
<controller type='pci' index='5' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='5' port='0xc'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x4'/>
</controller>
<controller type='pci' index='6' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='6' port='0xd'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x5'/>
</controller>
<controller type='pci' index='7' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='7' port='0xe'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x6'/>
</controller>
<controller type='pci' index='8' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='8' port='0xf'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x7'/>
</controller>
<controller type='pci' index='9' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='9' port='0x10'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='10' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='10' port='0x11'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
</controller>
<controller type='pci' index='11' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='11' port='0x12'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
</controller>
<controller type='pci' index='12' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='12' port='0x13'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
</controller>
<controller type='pci' index='13' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='13' port='0x14'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x4'/>
</controller>
<controller type='pci' index='14' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='14' port='0x15'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x5'/>
</controller>
<controller type='pci' index='15' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='15' port='0x16'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x6'/>
</controller>
<controller type='pci' index='16' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='16' port='0x17'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x7'/>
</controller>
<controller type='virtio-serial' index='0'>
<address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
</controller>
<interface type='network'>
<mac address='52:54:00:37:6a:d5'/>
<source network='default'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:db:35:b3'/>
<source network='default'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:8a:54:6c'/>
<source network='default'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:04:2b:e4'/>
<source network='default'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x07' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:6b:fc:ac'/>
<source network='default'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x08' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:d4:b0:e2'/>
<source network='default'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x09' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:52:2f:cc'/>
<source network='default'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x0a' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:0d:b1:bb'/>
<source network='default'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x0b' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:8a:a5:69'/>
<source network='default'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x0c' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:ad:60:b1'/>
<source network='default'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x0d' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:e3:9a:70'/>
<source network='default'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x0e' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:56:58:b3'/>
<source network='default'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x0f' slot='0x00' function='0x0'/>
</interface>
<serial type='pty'>
<target port='0'/>
</serial>
<console type='pty'>
<target type='serial' port='0'/>
</console>
<channel type='unix'>
<target type='virtio' name='org.qemu.guest_agent.0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
</devices>
</domain>
<domain type='kvm' id='57'>
<name>pci</name>
<uuid>5361dd27-bdc1-4178-b26f-323b4009b226</uuid>
<memory unit='KiB'>12312576</memory>
<currentMemory unit='KiB'>12312576</currentMemory>
<vcpu placement='static'>5</vcpu>
<resource>
<partition>/machine</partition>
</resource>
<os>
<type arch='aarch64' machine='virt-2.9'>hvm</type>
<loader readonly='yes' type='pflash'>/usr/share/AAVMF/AAVMF_CODE.fd</loader>
<nvram>/var/lib/libvirt/qemu/nvram/pci_VARS.fd</nvram>
<boot dev='hd'/>
</os>
<features>
<acpi/>
<gic version='2'/>
</features>
<cpu mode='host-passthrough' check='none'/>
<clock offset='utc'/>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<devices>
<emulator>/usr/bin/qemu-system-aarch64</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/home/linaro/virt/debian-30.qcow2'/>
<backingStore/>
<target dev='vda' bus='virtio'/>
<alias name='virtio-disk0'/>
<address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
</disk>
<controller type='scsi' index='0' model='virtio-scsi'>
<alias name='scsi0'/>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
</controller>
<controller type='pci' index='0' model='pcie-root'>
<alias name='pcie.0'/>
</controller>
<controller type='pci' index='1' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='1' port='0x8'/>
<alias name='pci.1'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='2' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='2' port='0x9'/>
<alias name='pci.2'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
</controller>
<controller type='pci' index='3' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='3' port='0xa'/>
<alias name='pci.3'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
</controller>
<controller type='pci' index='4' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='4' port='0xb'/>
<alias name='pci.4'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x3'/>
</controller>
<controller type='pci' index='5' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='5' port='0xc'/>
<alias name='pci.5'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x4'/>
</controller>
<controller type='pci' index='6' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='6' port='0xd'/>
<alias name='pci.6'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x5'/>
</controller>
<controller type='pci' index='7' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='7' port='0xe'/>
<alias name='pci.7'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x6'/>
</controller>
<controller type='pci' index='8' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='8' port='0xf'/>
<alias name='pci.8'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x7'/>
</controller>
<controller type='pci' index='9' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='9' port='0x10'/>
<alias name='pci.9'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='10' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='10' port='0x11'/>
<alias name='pci.10'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
</controller>
<controller type='pci' index='11' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='11' port='0x12'/>
<alias name='pci.11'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
</controller>
<controller type='pci' index='12' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='12' port='0x13'/>
<alias name='pci.12'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
</controller>
<controller type='pci' index='13' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='13' port='0x14'/>
<alias name='pci.13'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x4'/>
</controller>
<controller type='pci' index='14' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='14' port='0x15'/>
<alias name='pci.14'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x5'/>
</controller>
<controller type='pci' index='15' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='15' port='0x16'/>
<alias name='pci.15'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x6'/>
</controller>
<controller type='virtio-serial' index='0'>
<alias name='virtio-serial0'/>
<address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
</controller>
<interface type='network'>
<mac address='52:54:00:37:6a:d5'/>
<source network='default' bridge='virbr0'/>
<target dev='vnet0'/>
<model type='virtio'/>
<alias name='net0'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:db:35:b3'/>
<source network='default' bridge='virbr0'/>
<target dev='vnet1'/>
<model type='virtio'/>
<alias name='net1'/>
<address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:8a:54:6c'/>
<source network='default' bridge='virbr0'/>
<target dev='vnet4'/>
<model type='virtio'/>
<alias name='net2'/>
<address type='pci' domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:04:2b:e4'/>
<source network='default' bridge='virbr0'/>
<target dev='vnet5'/>
<model type='virtio'/>
<alias name='net3'/>
<address type='pci' domain='0x0000' bus='0x07' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:6b:fc:ac'/>
<source network='default' bridge='virbr0'/>
<target dev='vnet6'/>
<model type='virtio'/>
<alias name='net4'/>
<address type='pci' domain='0x0000' bus='0x08' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:d4:b0:e2'/>
<source network='default' bridge='virbr0'/>
<target dev='vnet7'/>
<model type='virtio'/>
<alias name='net5'/>
<address type='pci' domain='0x0000' bus='0x09' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:52:2f:cc'/>
<source network='default' bridge='virbr0'/>
<target dev='vnet8'/>
<model type='virtio'/>
<alias name='net6'/>
<address type='pci' domain='0x0000' bus='0x0a' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:0d:b1:bb'/>
<source network='default' bridge='virbr0'/>
<target dev='vnet9'/>
<model type='virtio'/>
<alias name='net7'/>
<address type='pci' domain='0x0000' bus='0x0b' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:8a:a5:69'/>
<source network='default' bridge='virbr0'/>
<target dev='vnet10'/>
<model type='virtio'/>
<alias name='net8'/>
<address type='pci' domain='0x0000' bus='0x0c' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:ad:60:b1'/>
<source network='default' bridge='virbr0'/>
<target dev='vnet11'/>
<model type='virtio'/>
<alias name='net9'/>
<address type='pci' domain='0x0000' bus='0x0d' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:e3:9a:70'/>
<source network='default' bridge='virbr0'/>
<target dev='vnet12'/>
<model type='virtio'/>
<alias name='net10'/>
<address type='pci' domain='0x0000' bus='0x0e' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:56:58:b3'/>
<source network='default' bridge='virbr0'/>
<target dev='vnet13'/>
<model type='virtio'/>
<alias name='net11'/>
<address type='pci' domain='0x0000' bus='0x0f' slot='0x00' function='0x0'/>
</interface>
<serial type='pty'>
<source path='/dev/pts/4'/>
<target port='0'/>
<alias name='serial0'/>
</serial>
<console type='pty' tty='/dev/pts/4'>
<source path='/dev/pts/4'/>
<target type='serial' port='0'/>
<alias name='serial0'/>
</console>
<channel type='unix'>
<source mode='bind' path='/var/lib/libvirt/qemu/channel/target/domain-57-pci/org.qemu.guest_agent.0'/>
<target type='virtio' name='org.qemu.guest_agent.0' state='disconnected'/>
<alias name='channel0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
</devices>
<seclabel type='none' model='none'/>
<seclabel type='dynamic' model='dac' relabel='yes'>
<label>+64055:+64055</label>
<imagelabel>+64055:+64055</imagelabel>
</seclabel>
</domain>
/usr/bin/qemu-system-aarch64 -name guest=pci,debug-threads=on -S -object
secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-108-pci/master-key.aes
-machine virt-2.9,accel=kvm,usb=off,dump-guest-core=off -cpu host -drive
file=/usr/share/AAVMF/AAVMF_CODE.fd,if=pflash,format=raw,unit=0,readonly=on
-drive file=/var/lib/libvirt/qemu/nvram/pci_VARS.fd,if=pflash,format=raw,unit=1
-m 12024 -realtime mlock=off -smp 5,sockets=5,cores=1,threads=1 -uuid
5361dd27-bdc1-4178-b26f-323b4009b226 -display none -no-user-config -nodefaults
-chardev
socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-108-pci/monitor.sock,server,nowait
-mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown
-boot strict=on -device
pcie-root-port,port=0x8,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x1
-device pcie-root-port,port=0x9,chassis=2,id=pci.2,bus=pcie.0,addr=0x1.0x1
-device pcie-root-port,port=0xa,chassis=3,id=pci.3,bus=pcie.0,addr=0x1.0x2
-device pcie-root-port,port=0xb,chassis=4,id=pci.4,bus=pcie.0,addr=0x1.0x3
-device pcie-root-port,port=0xc,chassis=5,id=pci.5,bus=pcie.0,addr=0x1.0x4
-device pcie-root-port,port=0xd,chassis=6,id=pci.6,bus=pcie.0,addr=0x1.0x5
-device pcie-root-port,port=0xe,chassis=7,id=pci.7,bus=pcie.0,addr=0x1.0x6
-device pcie-root-port,port=0xf,chassis=8,id=pci.8,bus=pcie.0,addr=0x1.0x7
-device
pcie-root-port,port=0x10,chassis=9,id=pci.9,bus=pcie.0,multifunction=on,addr=0x2
-device pcie-root-port,port=0x11,chassis=10,id=pci.10,bus=pcie.0,addr=0x2.0x1
-device pcie-root-port,port=0x12,chassis=11,id=pci.11,bus=pcie.0,addr=0x2.0x2
-device pcie-root-port,port=0x13,chassis=12,id=pci.12,bus=pcie.0,addr=0x2.0x3
-device pcie-root-port,port=0x14,chassis=13,id=pci.13,bus=pcie.0,addr=0x2.0x4
-device pcie-root-port,port=0x15,chassis=14,id=pci.14,bus=pcie.0,addr=0x2.0x5
-device pcie-root-port,port=0x16,chassis=15,id=pci.15,bus=pcie.0,addr=0x2.0x6
-device pcie-root-port,port=0x17,chassis=16,id=pci.16,bus=pcie.0,addr=0x2.0x7
-device virtio-scsi-pci,id=scsi0,bus=pci.2,addr=0x0 -device
virtio-serial-pci,id=virtio-serial0,bus=pci.3,addr=0x0 -drive
file=/home/linaro/virt/debian-30.qcow2,format=qcow2,if=none,id=drive-virtio-disk0
-device
virtio-blk-pci,scsi=off,bus=pci.4,addr=0x0,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1
-netdev tap,fd=27,id=hostnet0,vhost=on,vhostfd=29 -device
virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:37:6a:d5,bus=pci.1,addr=0x0
-netdev tap,fd=30,id=hostnet1,vhost=on,vhostfd=31 -device
virtio-net-pci,netdev=hostnet1,id=net1,mac=52:54:00:db:35:b3,bus=pci.5,addr=0x0
-netdev tap,fd=32,id=hostnet2,vhost=on,vhostfd=33 -device
virtio-net-pci,netdev=hostnet2,id=net2,mac=52:54:00:8a:54:6c,bus=pci.6,addr=0x0
-netdev tap,fd=34,id=hostnet3,vhost=on,vhostfd=35 -device
virtio-net-pci,netdev=hostnet3,id=net3,mac=52:54:00:04:2b:e4,bus=pci.7,addr=0x0
-netdev tap,fd=36,id=hostnet4,vhost=on,vhostfd=37 -device
virtio-net-pci,netdev=hostnet4,id=net4,mac=52:54:00:6b:fc:ac,bus=pci.8,addr=0x0
-netdev tap,fd=38,id=hostnet5,vhost=on,vhostfd=39 -device
virtio-net-pci,netdev=hostnet5,id=net5,mac=52:54:00:d4:b0:e2,bus=pci.9,addr=0x0
-netdev tap,fd=40,id=hostnet6,vhost=on,vhostfd=41 -device
virtio-net-pci,netdev=hostnet6,id=net6,mac=52:54:00:52:2f:cc,bus=pci.10,addr=0x0
-netdev tap,fd=42,id=hostnet7,vhost=on,vhostfd=43 -device
virtio-net-pci,netdev=hostnet7,id=net7,mac=52:54:00:0d:b1:bb,bus=pci.11,addr=0x0
-netdev tap,fd=44,id=hostnet8,vhost=on,vhostfd=45 -device
virtio-net-pci,netdev=hostnet8,id=net8,mac=52:54:00:8a:a5:69,bus=pci.12,addr=0x0
-netdev tap,fd=46,id=hostnet9,vhost=on,vhostfd=47 -device
virtio-net-pci,netdev=hostnet9,id=net9,mac=52:54:00:ad:60:b1,bus=pci.13,addr=0x0
-netdev tap,fd=48,id=hostnet10,vhost=on,vhostfd=49 -device
virtio-net-pci,netdev=hostnet10,id=net10,mac=52:54:00:e3:9a:70,bus=pci.14,addr=0x0
-netdev tap,fd=50,id=hostnet11,vhost=on,vhostfd=51 -device
virtio-net-pci,netdev=hostnet11,id=net11,mac=52:54:00:56:58:b3,bus=pci.15,addr=0x0
-chardev pty,id=charserial0 -serial chardev:charserial0 -chardev
socket,id=charchannel0,path=/var/lib/libvirt/qemu/channel/target/domain-108-pci/org.qemu.guest_agent.0,server,nowait
-device
virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0
-msg timestamp=on
/usr/bin/qemu-system-aarch64 -name guest=pci,debug-threads=on -S -object
secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-107-pci/master-key.aes
-machine virt-2.9,accel=kvm,usb=off,dump-guest-core=off -cpu host -drive
file=/usr/share/AAVMF/AAVMF_CODE.fd,if=pflash,format=raw,unit=0,readonly=on
-drive file=/var/lib/libvirt/qemu/nvram/pci_VARS.fd,if=pflash,format=raw,unit=1
-m 12024 -realtime mlock=off -smp 5,sockets=5,cores=1,threads=1 -uuid
5361dd27-bdc1-4178-b26f-323b4009b226 -display none -no-user-config -nodefaults
-chardev
socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-107-pci/monitor.sock,server,nowait
-mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown
-boot strict=on -device
pcie-root-port,port=0x8,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x1
-device pcie-root-port,port=0x9,chassis=2,id=pci.2,bus=pcie.0,addr=0x1.0x1
-device pcie-root-port,port=0xa,chassis=3,id=pci.3,bus=pcie.0,addr=0x1.0x2
-device pcie-root-port,port=0xb,chassis=4,id=pci.4,bus=pcie.0,addr=0x1.0x3
-device pcie-root-port,port=0xc,chassis=5,id=pci.5,bus=pcie.0,addr=0x1.0x4
-device pcie-root-port,port=0xd,chassis=6,id=pci.6,bus=pcie.0,addr=0x1.0x5
-device pcie-root-port,port=0xe,chassis=7,id=pci.7,bus=pcie.0,addr=0x1.0x6
-device pcie-root-port,port=0xf,chassis=8,id=pci.8,bus=pcie.0,addr=0x1.0x7
-device
pcie-root-port,port=0x10,chassis=9,id=pci.9,bus=pcie.0,multifunction=on,addr=0x2
-device pcie-root-port,port=0x11,chassis=10,id=pci.10,bus=pcie.0,addr=0x2.0x1
-device pcie-root-port,port=0x12,chassis=11,id=pci.11,bus=pcie.0,addr=0x2.0x2
-device pcie-root-port,port=0x13,chassis=12,id=pci.12,bus=pcie.0,addr=0x2.0x3
-device pcie-root-port,port=0x14,chassis=13,id=pci.13,bus=pcie.0,addr=0x2.0x4
-device pcie-root-port,port=0x15,chassis=14,id=pci.14,bus=pcie.0,addr=0x2.0x5
-device pcie-root-port,port=0x16,chassis=15,id=pci.15,bus=pcie.0,addr=0x2.0x6
-device virtio-scsi-pci,id=scsi0,bus=pci.2,addr=0x0 -device
virtio-serial-pci,id=virtio-serial0,bus=pci.3,addr=0x0 -drive
file=/home/linaro/virt/debian-30.qcow2,format=qcow2,if=none,id=drive-virtio-disk0
-device
virtio-blk-pci,scsi=off,bus=pci.4,addr=0x0,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1
-netdev tap,fd=28,id=hostnet0,vhost=on,vhostfd=30 -device
virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:37:6a:d5,bus=pci.1,addr=0x0
-netdev tap,fd=31,id=hostnet1,vhost=on,vhostfd=32 -device
virtio-net-pci,netdev=hostnet1,id=net1,mac=52:54:00:db:35:b3,bus=pci.5,addr=0x0
-netdev tap,fd=33,id=hostnet2,vhost=on,vhostfd=34 -device
virtio-net-pci,netdev=hostnet2,id=net2,mac=52:54:00:8a:54:6c,bus=pci.6,addr=0x0
-netdev tap,fd=35,id=hostnet3,vhost=on,vhostfd=36 -device
virtio-net-pci,netdev=hostnet3,id=net3,mac=52:54:00:04:2b:e4,bus=pci.7,addr=0x0
-netdev tap,fd=37,id=hostnet4,vhost=on,vhostfd=38 -device
virtio-net-pci,netdev=hostnet4,id=net4,mac=52:54:00:6b:fc:ac,bus=pci.8,addr=0x0
-netdev tap,fd=39,id=hostnet5,vhost=on,vhostfd=40 -device
virtio-net-pci,netdev=hostnet5,id=net5,mac=52:54:00:d4:b0:e2,bus=pci.9,addr=0x0
-netdev tap,fd=41,id=hostnet6,vhost=on,vhostfd=42 -device
virtio-net-pci,netdev=hostnet6,id=net6,mac=52:54:00:52:2f:cc,bus=pci.10,addr=0x0
-netdev tap,fd=43,id=hostnet7,vhost=on,vhostfd=44 -device
virtio-net-pci,netdev=hostnet7,id=net7,mac=52:54:00:0d:b1:bb,bus=pci.11,addr=0x0
-netdev tap,fd=45,id=hostnet8,vhost=on,vhostfd=46 -device
virtio-net-pci,netdev=hostnet8,id=net8,mac=52:54:00:8a:a5:69,bus=pci.12,addr=0x0
-netdev tap,fd=47,id=hostnet9,vhost=on,vhostfd=48 -device
virtio-net-pci,netdev=hostnet9,id=net9,mac=52:54:00:ad:60:b1,bus=pci.13,addr=0x0
-netdev tap,fd=49,id=hostnet10,vhost=on,vhostfd=50 -device
virtio-net-pci,netdev=hostnet10,id=net10,mac=52:54:00:e3:9a:70,bus=pci.14,addr=0x0
-netdev tap,fd=51,id=hostnet11,vhost=on,vhostfd=52 -device
virtio-net-pci,netdev=hostnet11,id=net11,mac=52:54:00:56:58:b3,bus=pci.15,addr=0x0
-chardev pty,id=charserial0 -serial chardev:charserial0 -chardev
socket,id=charchannel0,path=/var/lib/libvirt/qemu/channel/target/domain-107-pci/org.qemu.guest_agent.0,server,nowait
-device
virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0
-msg timestamp=on
