------- Comment From [email protected] 2016-09-13 16:35 EDT-------
Some additional information from our testing:
System was a Firestone / S822LC, 2 sockets, 8 cores each (POWER8), 64GB
memory, running Ubuntu 16.10 with SMT disabled (for KVM), using qemu-
system-common/qemu-system-pcc/qemu-kvm packages from Jon's PPA.
We tested 2 kernels, a local build of 4.7, and 4.4.0-30-generic from the
install ISO.
We tested physical function passthrough of 2 adapters, a Mellanox
ConnectX-3 and ConnectX-4
We don't have any GPUs on this system. I'm still trying to get some
results with GPUs, but we're having some issues with the machine we have
available (unrelated to the DDW code). From IBM's perspective however
the Mellanox test case is the more important driver of the DDW
work/feature request so our hope is verification against Mellanox
adapters is enough confirm the DDW functionality is working as intended,
and that any additional requirements for GPU passthrough be handled as
separate issues.
Test steps are below (retried for each adapter/kernel):
# Bind device to vfio-pci module
root@ltc-fire1:~/mdroth# lspci | grep Mellanox
0001:01:00.0 Ethernet controller: Mellanox Technologies MT27700 Family
[ConnectX-4]
0001:01:00.1 Ethernet controller: Mellanox Technologies MT27700 Family
[ConnectX-4]
0004:01:00.0 Ethernet controller: Mellanox Technologies MT27500 Family
[ConnectX-3]
root@ltc-fire1:~/mdroth# echo 0004:01:00.0
>/sys/bus/pci/devices/0004:01:00.0/driver/unbind
root@ltc-fire1:~/mdroth# echo 0x15b3 0x1003 >/sys/bus/pci/drivers/vfio-pci/
bind module/ new_id remove_id uevent unbind
root@ltc-fire1:~/mdroth# echo 0004:01:00.0 >/sys/bus/pci/drivers/vfio-pci/bind
# Start guest
root@ltc-fire1:~/mdroth# qemu-system-ppc64 --smp
sockets=1,cores=2,threads=8 --enable-kvm -M pseries -m
3G,slots=32,maxmem=8G -device virtio-blk-pci,id=disk1,drive=drive1
-drive
file=ubuntu16.04.base.img.3,aio=threads,cache=none,if=none,id=drive1
-device spapr-vscsi,id=scsi0,reg=0x2000 -monitor unix:/tmp/mdroth-
vm3-hmp.sock,server,nowait -qmp unix:/tmp/mdroth-
vm3-qmp.sock,server,nowait -device virtio-net-
pci,mac=52:54:00:12:34:01,netdev=net1,id=nic1 -netdev
user,hostfwd=tcp:0.0.0.0:2223-:22,id=net1 -device vfio-
pci,host=0004:01:00.0,id=vfio0 -L /root/mdroth-qemu/pc-bios -boot c -vga
none -nographic
# Verify that DDW window ("window#1") gets created after guest kernel
boots and passthrough device driver enables 64-bit DMA
root@ltc-fire1:~/mdroth# dmesg | tail -9
[ 5801.875253] pci 0004:01 : [PE# 000] Disabling 64-bit DMA bypass
[ 5801.875262] pci 0004:01 : [PE# 000] Removing DMA window #0
[ 5801.875840] pci 0004:01 : [PE# 000] Setting up window#0 0..7fffffff
pg=1000
[ 5802.053473] pci 0004:01 : [PE# 000] Removing DMA window #0
[ 5802.101984] vfio-pci 0004:01:00.0: enabling device (0400 -> 0402)
[ 5806.123998] vfio_ecap_init: 0004:01:00.0 hiding ecap 0x19@0x18c
[ 5806.137354] KVM guest htab at c000000f2b000000 (order 26), LPID 1
[ 5810.159668] pci 0004:01 : [PE# 000] Setting up window#0 0..3fffffff
pg=1000
[ 5862.518987] pci 0004:01 : [PE# 000] Setting up window#1
800000000000000..8000001ffffffff pg=10000
# Shutdown guest, verify that windows are destroyed and default 32-bit
window is recreated
root@ltc-fire1:~/mdroth# dmesg | tail -4
[ 6187.635524] pci 0004:01 : [PE# 000] Removing DMA window #0
[ 6187.635533] pci 0004:01 : [PE# 000] Removing DMA window #1
[ 6187.641791] pci 0004:01 : [PE# 000] Setting up window#0 0..7fffffff
pg=1000
[ 6187.641795] pci 0004:01 : [PE# 000] Enabling 64-bit DMA bypass
--
You received this bug notification because you are a member of Ubuntu
Bugs, which is subscribed to Ubuntu.
https://bugs.launchpad.net/bugs/1541902
Title:
Enable DDW on qemu aiming GPU passthrough performance on Ubuntu KVM
To manage notifications about this bug go to:
https://bugs.launchpad.net/ubuntu/+source/qemu/+bug/1541902/+subscriptions
--
ubuntu-bugs mailing list
[email protected]
https://lists.ubuntu.com/mailman/listinfo/ubuntu-bugs
