Hi Everybody,
I'm using a Mellanox Connectx4-Ln NIC (OFED 4.3-1.0.1.0) with ovs-2.9.0,
dpdk 17.11 and qemu 3.0. I've followed the ovs documentation [1] and
mellanox documentation [2] to configure ovs, the dpdk ports connected to
the physical ports and the dpdkvhostuserclient ports for the VM. I also
inserted appropriate flow rules to send the packets to the VM. However,
I do not receive any packets in the VM. Packets are sent and received if
I simply send the packets from one dpdk port to the other (phy-phy). So
I'm sure there is something strange going on when sending the packets
from the dpdk port to the dpdkvhostuserclient port.
Below is some configuration/system information. Any help is much
appreciated.
**************************************************************************************
Interface/port/flow configuration
root@havel:/usr/local/src/ovs-dpdk/openvswitch-2.9.0/utilities#
./ovs-vsctl add-port br0 dpdk0 -- set Interface dpdk0 type=dpdk
options:dpdk-devargs=0000:03:00.0,n_rxq_desc=1024,n_txq_desc=1024,n_rxq=1,pmd-rxq-affinity="2"
ofport_request=1
root@havel:/usr/local/src/ovs-dpdk/openvswitch-2.9.0/utilities#
./ovs-vsctl add-port br0 dpdk1 -- set Interface dpdk1 type=dpdk
options:dpdk-devargs=0000:03:00.1,n_rxq_desc=1024,n_txq_desc=1024,n_rxq=1,pmd-rxq-affinity="2"
ofport_request=2
root@havel:/usr/local/src/ovs-dpdk/openvswitch-2.9.0/utilities#
./ovs-vsctl add-port br0 vhost-user1 -- set Interface vhost-user1
type=dpdkvhostuserclient options:vhost-server-path=/tmp/vhost-user1
root@havel:/usr/local/src/ovs-dpdk/openvswitch-2.9.0/utilities#
./ovs-vsctl add-port br0 vhost-user2 -- set Interface vhost-user2
type=dpdkvhostuserclient options:vhost-server-path=/tmp/vhost-user2
root@havel:/usr/local/src/ovs-dpdk/openvswitch-2.9.0/utilities#
./ovs-vsctl show
3fcbc293-ad6d-4f5c-be97-5236d75ce47a
Bridge "br0"
Port "dpdk0"
Interface "dpdk0"
type: dpdk
options:
{dpdk-devargs="0000:03:00.0,n_rxq_desc=1024,n_txq_desc=1024,n_rxq=1,pmd-rxq-affinity=2"}
Port "dpdk1"
Interface "dpdk1"
type: dpdk
options:
{dpdk-devargs="0000:03:00.1,n_rxq_desc=1024,n_txq_desc=1024,n_rxq=1,pmd-rxq-affinity=2"}
Port "br0"
Interface "br0"
type: internal
Port "vhost-user2"
Interface "vhost-user2"
type: dpdkvhostuserclient
options: {vhost-server-path="/tmp/vhost-user2"}
Port "vhost-user1"
Interface "vhost-user1"
type: dpdkvhostuserclient
options: {vhost-server-path="/tmp/vhost-user1"}
root@havel:/usr/local/src/ovs-dpdk/openvswitch-2.9.0/utilities#
./ovs-vsctl list interface dpdk0
_uuid : 501edcef-76f6-4909-8eb8-df0f62ba3203
admin_state : up
bfd : {}
bfd_status : {}
cfm_fault : []
cfm_fault_status : []
cfm_flap_count : []
cfm_health : []
cfm_mpid : []
cfm_remote_mpids : []
cfm_remote_opstate : []
duplex : full
error : []
external_ids : {}
ifindex : 10816857
ingress_policing_burst: 0
ingress_policing_rate: 0
lacp_current : []
link_resets : 0
link_speed : 1000000000
link_state : up
lldp : {}
mac : []
mac_in_use : "ec:0d:9a:cc:a5:2a"
mtu : 1500
mtu_request : []
name : "dpdk0"
ofport : 1
ofport_request : 1
options :
{dpdk-devargs="0000:03:00.0,n_rxq_desc=1024,n_txq_desc=1024,n_rxq=1,pmd-rxq-affinity=2"}
other_config : {}
statistics : {rx_bytes=6523520, rx_dropped=0, rx_errors=0,
rx_mbuf_allocation_errors=0, rx_missed_errors=0, rx_packets=101930,
tx_bytes=0, tx_dropped=0, tx_errors=0, tx_packets=0}
status : {driver_name="net_mlx5", if_descr="DPDK 17.11.1
net_mlx5", if_type="6", max_hash_mac_addrs="0", max_mac_addrs="128",
max_rx_pktlen="1518", max_rx_queues="65535", max_tx_queues="65535",
max_vfs="0", max_vmdq_pools="0", min_rx_bufsize="32", numa_id="0",
pci-device_id="0x1015", pci-vendor_id="0x5555", port_no="0"}
type : dpdk
root@havel:/usr/local/src/ovs-dpdk/openvswitch-2.9.0/utilities#
./ovs-vsctl list interface dpdk1
_uuid : 93589e22-b258-4d15-96de-cf8cac4f2816
admin_state : up
bfd : {}
bfd_status : {}
cfm_fault : []
cfm_fault_status : []
cfm_flap_count : []
cfm_health : []
cfm_mpid : []
cfm_remote_mpids : []
cfm_remote_opstate : []
duplex : full
error : []
external_ids : {}
ifindex : 6054325
ingress_policing_burst: 0
ingress_policing_rate: 0
lacp_current : []
link_resets : 0
link_speed : 1000000000
link_state : up
lldp : {}
mac : []
mac_in_use : "ec:0d:9a:cc:a5:2b"
mtu : 1500
mtu_request : []
name : "dpdk1"
ofport : 2
ofport_request : 2
options :
{dpdk-devargs="0000:03:00.1,n_rxq_desc=1024,n_txq_desc=1024,n_rxq=1,pmd-rxq-affinity=2"}
other_config : {}
statistics : {rx_bytes=0, rx_dropped=0, rx_errors=0,
rx_mbuf_allocation_errors=0, rx_missed_errors=0, rx_packets=0,
tx_bytes=0, tx_dropped=0, tx_errors=0, tx_packets=0}
status : {driver_name="net_mlx5", if_descr="DPDK 17.11.1
net_mlx5", if_type="6", max_hash_mac_addrs="0", max_mac_addrs="128",
max_rx_pktlen="1518", max_rx_queues="65535", max_tx_queues="65535",
max_vfs="0", max_vmdq_pools="0", min_rx_bufsize="32", numa_id="0",
pci-device_id="0x1015", pci-vendor_id="0x5555", port_no="1"}
type : dpdk
**************************************************************************************
As you can see there are no packets received on the vhost-user2 port
which the VM is supposed to send.
root@havel:/usr/local/src/ovs-dpdk/openvswitch-2.9.0/utilities#
./ovs-ofctl dump-flows br0
cookie=0x0, duration=167.756s, table=0, n_packets=101930,
n_bytes=6115800, in_port=dpdk0 actions=output:"vhost-user1"
cookie=0x0, duration=164.631s, table=0, n_packets=0, n_bytes=0,
in_port="vhost-user2" actions=output:dpdk1
**************************************************************************************
ovs-vswitchd log set to INFO for dpdk
2018-08-28T16:36:40.153Z|00001|dpdk|INFO|VHOST_CONFIG: /tmp/vhost-user1:
connected
2018-08-28T16:36:40.153Z|00002|dpdk|INFO|VHOST_CONFIG: new device,
handle is 0
2018-08-28T16:36:40.153Z|00003|dpdk|INFO|VHOST_CONFIG: /tmp/vhost-user2:
connected
2018-08-28T16:36:40.153Z|00004|dpdk|INFO|VHOST_CONFIG: new device,
handle is 1
2018-08-28T16:36:40.228Z|00001|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_GET_FEATURES
2018-08-28T16:36:40.228Z|00002|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_GET_PROTOCOL_FEATURES
2018-08-28T16:36:40.228Z|00003|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_PROTOCOL_FEATURES
2018-08-28T16:36:40.228Z|00004|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_GET_QUEUE_NUM
2018-08-28T16:36:40.228Z|00005|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_OWNER
2018-08-28T16:36:40.228Z|00006|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_GET_FEATURES
2018-08-28T16:36:40.228Z|00007|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_VRING_CALL
2018-08-28T16:36:40.228Z|00008|dpdk|INFO|VHOST_CONFIG: vring call idx:0
file:52
2018-08-28T16:36:40.228Z|00009|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_VRING_CALL
2018-08-28T16:36:40.228Z|00010|dpdk|INFO|VHOST_CONFIG: vring call idx:1
file:53
2018-08-28T16:36:40.229Z|00011|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_GET_FEATURES
2018-08-28T16:36:40.229Z|00012|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_GET_PROTOCOL_FEATURES
2018-08-28T16:36:40.229Z|00013|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_PROTOCOL_FEATURES
2018-08-28T16:36:40.229Z|00014|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_GET_QUEUE_NUM
2018-08-28T16:36:40.229Z|00015|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_OWNER
2018-08-28T16:36:40.229Z|00016|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_GET_FEATURES
2018-08-28T16:36:40.229Z|00017|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_VRING_CALL
2018-08-28T16:36:40.229Z|00018|dpdk|INFO|VHOST_CONFIG: vring call idx:0
file:54
2018-08-28T16:36:40.229Z|00019|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_VRING_CALL
2018-08-28T16:36:40.229Z|00020|dpdk|INFO|VHOST_CONFIG: vring call idx:1
file:55
2018-08-28T16:36:54.707Z|00021|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_VRING_ENABLE
2018-08-28T16:36:54.707Z|00022|dpdk|INFO|VHOST_CONFIG: set queue enable:
1 to qp idx: 0
2018-08-28T16:36:54.707Z|00023|netdev_dpdk|INFO|State of queue 0 (
tx_qid 0 ) of vhost device '/tmp/vhost-user1'changed to 'enabled'
2018-08-28T16:36:54.707Z|00024|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_VRING_ENABLE
2018-08-28T16:36:54.707Z|00025|dpdk|INFO|VHOST_CONFIG: set queue enable:
1 to qp idx: 1
2018-08-28T16:36:54.709Z|00026|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_FEATURES
2018-08-28T16:36:54.710Z|00027|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_VRING_ENABLE
2018-08-28T16:36:54.710Z|00028|dpdk|INFO|VHOST_CONFIG: set queue enable:
1 to qp idx: 0
2018-08-28T16:36:54.710Z|00029|netdev_dpdk|INFO|State of queue 0 (
tx_qid 0 ) of vhost device '/tmp/vhost-user2'changed to 'enabled'
2018-08-28T16:36:54.710Z|00030|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_VRING_ENABLE
2018-08-28T16:36:54.710Z|00031|dpdk|INFO|VHOST_CONFIG: set queue enable:
1 to qp idx: 1
2018-08-28T16:36:54.710Z|00032|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_FEATURES
2018-08-28T16:39:08.587Z|00033|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_VRING_ENABLE
2018-08-28T16:39:08.587Z|00034|dpdk|INFO|VHOST_CONFIG: set queue enable:
1 to qp idx: 0
2018-08-28T16:39:08.587Z|00035|netdev_dpdk|INFO|State of queue 0 (
tx_qid 0 ) of vhost device '/tmp/vhost-user1'changed to 'enabled'
2018-08-28T16:39:08.587Z|00036|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_VRING_ENABLE
2018-08-28T16:39:08.587Z|00037|dpdk|INFO|VHOST_CONFIG: set queue enable:
1 to qp idx: 1
2018-08-28T16:39:08.587Z|00038|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_FEATURES
2018-08-28T16:39:08.591Z|00039|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_VRING_ENABLE
2018-08-28T16:39:08.591Z|00040|dpdk|INFO|VHOST_CONFIG: set queue enable:
1 to qp idx: 0
2018-08-28T16:39:08.591Z|00041|netdev_dpdk|INFO|State of queue 0 (
tx_qid 0 ) of vhost device '/tmp/vhost-user2'changed to 'enabled'
2018-08-28T16:39:08.591Z|00042|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_VRING_ENABLE
2018-08-28T16:39:08.591Z|00043|dpdk|INFO|VHOST_CONFIG: set queue enable:
1 to qp idx: 1
2018-08-28T16:39:08.591Z|00044|dpdk|INFO|VHOST_CONFIG: read message
VHOST_USER_SET_FEATURES
2018-08-28T16:39:19.435Z|00025|connmgr|INFO|br0<->unix#5: 1 flow_mods in
the last 0 s (1 adds)
2018-08-28T16:39:26.619Z|00026|connmgr|INFO|br0<->unix#7: 1 flow_mods in
the last 0 s (1 adds)
**************************************************************************************
On the guest VM I load the driver and bind the pci addresses to the
igb_uio driver and then
start the l2fwd dpdk app. However, as you can see there are 0 packets
received and sent.
root@guestVM:/usr/local/src/dpdk/dpdk-stable-17.11.1# modprobe uio
root@guestVM:/usr/local/src/dpdk/dpdk-stable-17.11.1# insmod
x86_64-native-linuxapp-gcc/kmod/igb_uio.ko
root@guestVM:/usr/local/src/dpdk/dpdk-stable-17.11.1#
./usertools/dpdk-devbind.py --bind=igb_uio 00:08.0
root@guestVM:/usr/local/src/dpdk/dpdk-stable-17.11.1#
./usertools/dpdk-devbind.py --bind=igb_uio 00:09.0
root@guestVM:/usr/local/src/dpdk/dpdk-stable-17.11.1#
./usertools/dpdk-devbind.py --status
Network devices using DPDK-compatible driver
============================================
0000:00:08.0 'Virtio network device 1000' drv=igb_uio unused=
0000:00:09.0 'Virtio network device 1000' drv=igb_uio unused=
Network devices using kernel driver
===================================
0000:00:03.0 'RTL-8100/8101L/8139 PCI Fast Ethernet Adapter 8139'
if=ens3 drv=8139cp unused=8139too,igb_uio *Active*
Other Network devices
=====================
<none>
Crypto devices using DPDK-compatible driver
===========================================
<none>
Crypto devices using kernel driver
==================================
<none>
Other Crypto devices
====================
<none>
Eventdev devices using DPDK-compatible driver
=============================================
<none>
Eventdev devices using kernel driver
====================================
<none>
Other Eventdev devices
======================
<none>
Mempool devices using DPDK-compatible driver
============================================
<none>
Mempool devices using kernel driver
===================================
<none>
Other Mempool devices
=====================
<none>
root@guestVM:/usr/local/src/dpdk/dpdk-stable-17.11.1/examples/l2fwd/build#
./l2fwd -l 0-1 -n 2 -- -q 1 -p 0x0003
EAL: Detected 2 lcore(s)
EAL: No free hugepages reported in hugepages-1048576kB
EAL: Probing VFIO support...
EAL: WARNING: cpu flags constant_tsc=yes nonstop_tsc=no -> using
unreliable clock cycles !
EAL: PCI device 0000:00:08.0 on NUMA socket -1
EAL: Invalid NUMA socket, default to 0
EAL: probe driver: 1af4:1000 net_virtio
EAL: PCI device 0000:00:09.0 on NUMA socket -1
EAL: Invalid NUMA socket, default to 0
EAL: probe driver: 1af4:1000 net_virtio
MAC updating enabled
Lcore 0: RX port 0
Lcore 1: RX port 1
Initializing port 0... done:
Port 0, MAC address: 52:54:00:E1:B4:E9
Initializing port 1... done:
Port 1, MAC address: 52:54:00:A2:06:AE
Checking link statusdone
Port0 Link Up. Speed 10000 Mbps - full-duplex
Port1 Link Up. Speed 10000 Mbps - full-duplex
L2FWD: entering main loop on lcore 1
L2FWD: -- lcoreid=1 portid=1
L2FWD: entering main loop on lcore 0
L2FWD: -- lcoreid=0 portid=0
Port statistics ====================================
Statistics for port 0 ------------------------------
Packets sent: 0
Packets received: 0
Packets dropped: 0
Statistics for port 1 ------------------------------
Packets sent: 0
Packets received: 0
Packets dropped: 0
Aggregate statistics ===============================
Total packets sent: 0
Total packets received: 0
Total packets dropped: 0
====================================================
**************************************************************************************
Here is the Guest VM xml used for the vm.
<domain type='kvm' id='1'>
<name>tenant-green-1</name>
<uuid>6de25865-2540-41a6-8ec9-588862c1b2aa</uuid>
<memory unit='KiB'>4194304</memory>
<currentMemory unit='KiB'>4194304</currentMemory>
<vcpu placement='static'>2</vcpu>
<cputune>
<vcpupin vcpu='0' cpuset='3'/>
<vcpupin vcpu='1' cpuset='4'/>
</cputune>
<resource>
<partition>/machine</partition>
</resource>
<os>
<type arch='x86_64' machine='pc-i440fx-xenial'>hvm</type>
<boot dev='hd'/>
</os>
<features>
<acpi/>
<apic/>
</features>
<cpu mode='host-model'>
<model fallback='allow'/>
</cpu>
<clock offset='utc'>
<timer name='rtc' tickpolicy='catchup'/>
<timer name='pit' tickpolicy='delay'/>
<timer name='hpet' present='no'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>restart</on_crash>
<pm>
<suspend-to-mem enabled='no'/>
<suspend-to-disk enabled='no'/>
</pm>
<devices>
<emulator>/usr/bin/kvm-spice</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/var/lib/libvirt/images/tenant-green-1.qcow2'/>
<backingStore/>
<target dev='hda' bus='ide'/>
<alias name='ide0-0-0'/>
<address type='drive' controller='0' bus='0' target='0' unit='0'/>
</disk>
<controller type='usb' index='0' model='ich9-ehci1'>
<alias name='usb'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06'
function='0x7'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci1'>
<alias name='usb'/>
<master startport='0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06'
function='0x0' multifunction='on'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci2'>
<alias name='usb'/>
<master startport='2'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06'
function='0x1'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci3'>
<alias name='usb'/>
<master startport='4'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06'
function='0x2'/>
</controller>
<controller type='pci' index='0' model='pci-root'>
<alias name='pci.0'/>
</controller>
<controller type='ide' index='0'>
<alias name='ide'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01'
function='0x1'/>
</controller>
<controller type='virtio-serial' index='0'>
<alias name='virtio-serial0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05'
function='0x0'/>
</controller>
<interface type='network'>
<mac address='52:54:00:70:f3:7f'/>
<source network='default' bridge='virbr0'/>
<target dev='vnet0'/>
<model type='rtl8139'/>
<alias name='net0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03'
function='0x0'/>
</interface>
<interface type='vhostuser'>
<mac address='52:54:00:e1:b4:e9'/>
<source type='unix' path='/tmp/vhost-user1' mode='server'/>
<model type='virtio'/>
<driver name='vhost'/>
<alias name='net1'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x08'
function='0x0'/>
</interface>
<interface type='vhostuser'>
<mac address='52:54:00:a2:06:ae'/>
<source type='unix' path='/tmp/vhost-user2' mode='server'/>
<model type='virtio'/>
<driver name='vhost'/>
<alias name='net2'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x09'
function='0x0'/>
</interface>
<serial type='pty'>
<source path='/dev/pts/6'/>
<target port='0'/>
<alias name='serial0'/>
</serial>
<console type='pty' tty='/dev/pts/6'>
<source path='/dev/pts/6'/>
<target type='serial' port='0'/>
<alias name='serial0'/>
</console>
<channel type='spicevmc'>
<target type='virtio' name='com.redhat.spice.0' state='disconnected'/>
<alias name='channel0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<graphics type='spice' port='5900' autoport='yes' listen='127.0.0.1'>
<listen type='address' address='127.0.0.1'/>
<image compression='off'/>
</graphics>
<sound model='ich6'>
<alias name='sound0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04'
function='0x0'/>
</sound>
<video>
<model type='qxl' ram='65536' vram='65536' vgamem='16384' heads='1'/>
<alias name='video0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02'
function='0x0'/>
</video>
<redirdev bus='usb' type='spicevmc'>
<alias name='redir0'/>
</redirdev>
<redirdev bus='usb' type='spicevmc'>
<alias name='redir1'/>
</redirdev>
<memballoon model='virtio'>
<alias name='balloon0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07'
function='0x0'/>
</memballoon>
</devices>
<seclabel type='dynamic' model='apparmor' relabel='yes'>
<label>libvirt-6de25865-2540-41a6-8ec9-588862c1b2aa</label>
<imagelabel>libvirt-6de25865-2540-41a6-8ec9-588862c1b2aa</imagelabel>
</seclabel>
</domain>
**************************************************************************************
[1] http://docs.openvswitch.org/en/latest/topics/dpdk/vhost-user/
[2] https://community.mellanox.com/docs/DOC-3073
--
Thanks,
Kashyap Thimmaraju <[email protected]>
Security in Telecommunications <sect.tu-berlin.de>
Technische Universität Berlin
Ernst-Reuter-Platz 7, Sekr TEL 17
10587 Berlin, Germany
Phone: +49 30 8353 58351
_______________________________________________
discuss mailing list
[email protected]
https://mail.openvswitch.org/mailman/listinfo/ovs-discuss
