Hi guys，
I meet a question about ovs dpdk multiple queues.
When testing tcp band width with iperf3 or qperf ,all packets are only
in one rxq.And the bw result is too low.
Would you please help me?
Version
ovs-vsctl (Open vSwitch) 2.11.0
DPDK 18.11.9
Configration
[root@controller-node openvswitch-2.11.0]# ovs-vsctl list open_vswitch
_uuid : 154520f8-47b1-4dbb-ade1-650a9dac8340
bridges : [3878e475-e959-4a6d-82f8-b931d8cd43d5,
8f1e93a1-f844-4027-97fa-276e32a9b555, 9d08bcc7-cfb7-434b-8247-7b6c02487e07]
cur_cfg : 151
datapath_types : [netdev, system]
db_version : []
dpdk_initialized : true
dpdk_version : "DPDK 18.11.9"
external_ids : {hostname=controller-node}
iface_types : [dpdk, dpdkr, dpdkvhostuser, dpdkvhostuserclient, erspan,
geneve, gre, internal, "ip6erspan", "ip6gre", lisp, patch, stt, system, tap,
vxlan]
manager_options : []
next_cfg : 151
other_config : {dpdk-init="true", dpdk-socket-mem="4096,0",
pmd-auto-lb="true", pmd-auto-lb-rebal-interval="10", pmd-cpu-mask="0xf000000",
pmd-perf-metrics="true", pmd-rxq-assign=cycles, vhost-iommu-support="true",
vhost-sock-dir="."}
ovs_version : []
ssl : []
statistics : {}
system_type : []
system_version : []
[root@controller-node openvswitch-2.11.0]# ovs-vsctl list Interface dpdk0
_uuid : 70193180-a43b-4c5a-9013-f9004f20950a
admin_state : up
bfd : {}
bfd_status : {}
cfm_fault : []
cfm_fault_status : []
cfm_flap_count : []
cfm_health : []
cfm_mpid : []
cfm_remote_mpids : []
cfm_remote_opstate : []
duplex : full
error : []
external_ids : {}
ifindex : 10816857
ingress_policing_burst: 0
ingress_policing_rate: 0
lacp_current : []
link_resets : 2
link_speed : 10000000000
link_state : up
lldp : {}
mac : []
mac_in_use : "b4:96:91:49:56:28"
mtu : 1500
mtu_request : 1500
name : "dpdk0"
ofport : 1
ofport_request : []
options : {dpdk-devargs="0000:02:00.0", n_rxq="4",
n_rxq_desc="2048", n_txq="1"}
other_config : {}
statistics : {flow_director_filter_add_errors=164803,
flow_director_filter_remove_errors=456422, mac_local_errors=0,
mac_remote_errors=0, "rx_128_to_255_packets"=29913, "rx_1_to_64_packets"=23786,
"rx_256_to_511_packets"=23611, "rx_512_to_1023_packets"=10639,
"rx_65_to_127_packets"=1288610, rx_broadcast_packets=10181,
rx_bytes=271122791632, rx_crc_errors=26168302729, rx_dropped=0, rx_errors=0,
rx_fcoe_crc_errors=68, rx_fcoe_dropped=0, rx_fcoe_mbuf_allocation_errors=0,
rx_fragment_errors=0, rx_illegal_byte_errors=0, rx_jabber_errors=0,
rx_length_errors=0, rx_mac_short_packet_dropped=45097959,
rx_management_dropped=10639, rx_management_packets=23611,
rx_mbuf_allocation_errors=0, rx_missed_errors=0, rx_oversize_errors=1288610,
rx_packets=180342635, "rx_priority0_dropped"=0,
"rx_priority0_mbuf_allocation_errors"=0, "rx_priority1_dropped"=0,
"rx_priority1_mbuf_allocation_errors"=16472, "rx_priority2_dropped"=0,
"rx_priority2_mbuf_allocation_errors"=0, "rx_priority3_dropped"=0,
"rx_priority3_mbuf_allocation_errors"=0, "rx_priority4_dropped"=0,
"rx_priority4_mbuf_allocation_errors"=0, "rx_priority5_dropped"=0,
"rx_priority5_mbuf_allocation_errors"=0, "rx_priority6_dropped"=0,
"rx_priority6_mbuf_allocation_errors"=0, "rx_priority7_dropped"=0,
"rx_priority7_mbuf_allocation_errors"=0, rx_undersize_errors=23786,
"tx_128_to_255_packets"=1174316, "tx_1_to_64_packets"=119,
"tx_256_to_511_packets"=164803, "tx_512_to_1023_packets"=456422,
"tx_65_to_127_packets"=28554784, tx_broadcast_packets=68, tx_bytes=26168304871,
tx_dropped=0, tx_errors=0, tx_management_packets=178966076,
tx_multicast_packets=7, tx_packets=45097959}
status : {driver_name=net_ixgbe, if_descr="DPDK 18.11.9
net_ixgbe", if_type="6", link_speed="10Gbps", max_hash_mac_addrs="4096",
max_mac_addrs="127", max_rx_pktlen="1518", max_rx_queues="128",
max_tx_queues="64", max_vfs="0", max_vmdq_pools="64", min_rx_bufsize="1024",
numa_id="0", pci-device_id="0x1528", pci-vendor_id="0x8086", port_no="0"}
type : dpdk
[root@controller-node openvswitch-2.11.0]# ovs-vsctl list Interface vhost-user1
_uuid : 0cc484b0-39a8-4df4-83fe-50eb629f96bd
admin_state : up
bfd : {}
bfd_status : {}
cfm_fault : []
cfm_fault_status : []
cfm_flap_count : []
cfm_health : []
cfm_mpid : []
cfm_remote_mpids : []
cfm_remote_opstate : []
duplex : []
error : []
external_ids : {}
ifindex : 16697981
ingress_policing_burst: 0
ingress_policing_rate: 0
lacp_current : []
link_resets : 0
link_speed : []
link_state : up
lldp : {}
mac : []
mac_in_use : "00:00:00:00:00:00"
mtu : 1500
mtu_request : []
name : "vhost-user1"
ofport : 3
ofport_request : []
options : {n_rxq="4",
vhost-server-path="/usr/local/var/run/openvswitch/vhost-user1"}
other_config : {}
statistics : {"rx_1024_to_1522_packets"=14698328,
"rx_128_to_255_packets"=29260, "rx_1523_to_max_packets"=0,
"rx_1_to_64_packets"=23111, "rx_256_to_511_packets"=106474,
"rx_512_to_1023_packets"=162651, "rx_65_to_127_packets"=29612807,
rx_bytes=23650574223, rx_dropped=0, rx_errors=0, rx_packets=44632631,
tx_bytes=262244518043, tx_dropped=736, tx_packets=180496251}
status : {features="0x0000000170606782", mode=client,
num_of_vrings="8", numa="1",
socket="/usr/local/var/run/openvswitch/vhost-user1", status=connected,
"vring_0_size"="1024", "vring_1_size"="1024", "vring_2_size"="1024",
"vring_3_size"="1024", "vring_4_size"="1024", "vring_5_size"="1024",
"vring_6_size"="1024", "vring_7_size"="1024"}
type : dpdkvhostuserclient
[root@controller-node openvswitch-2.11.0]# ovs-vsctl show
154520f8-47b1-4dbb-ade1-650a9dac8340
Bridge br-int
Port patch-tun
Interface patch-tun
type: patch
options: {peer=patch-int}
Port br-int
Interface br-int
type: internal
Port int-br-ex
Interface int-br-ex
type: patch
options: {peer=phy-br-ex}
Port "vhost-user1"
Interface "vhost-user1"
type: dpdkvhostuserclient
options: {n_rxq="4",
vhost-server-path="/usr/local/var/run/openvswitch/vhost-user1"}
Bridge br-ex
Port phy-br-ex
Interface phy-br-ex
type: patch
options: {peer=int-br-ex}
Port br-ex
Interface br-ex
type: internal
Port "dpdk0"
Interface "dpdk0"
type: dpdk
options: {dpdk-devargs="0000:02:00.0", n_rxq="4",
n_rxq_desc="2048", n_txq="1"}
Bridge br-tun
Port "vxlan-0adcb786"
Interface "vxlan-0adcb786"
type: vxlan
options: {df_default="true", egress_pkt_mark="0", in_key=flow,
local_ip="10.220.147.157", out_key=flow, remote_ip="10.220.147.172"}
Port br-tun
Interface br-tun
type: internal
Port "vxlan-0adcb78b"
Interface "vxlan-0adcb78b"
type: vxlan
options: {df_default="true", egress_pkt_mark="0", in_key=flow,
local_ip="10.220.147.157", out_key=flow, remote_ip="10.220.183.139"}
Port patch-int
Interface patch-int
type: patch
options: {peer=patch-tun}
qemu-kvm -name $VM_NAME -cpu host -enable-kvm \
-m $GUEST_MEM -drive file=$QCOW2_IMAGE --nographic -snapshot \
-numa node,memdev=mem -mem-prealloc -smp 4,sockets=1,cores=2,threads=2 \
-object
memory-backend-file,id=mem,size=$GUEST_MEM,mem-path=/dev/hugepages,share=on \
-chardev socket,id=char0,path=$VHOST_SOCK_DIR/vhost-user1,server \
-netdev type=vhost-user,id=mynet1,chardev=char0,vhostforce,queues=4 \
-device
virtio-net-pci,host_mtu=1450,rx_queue_size=1024,tx_queue_size=1024,mac=fa:16:3e:ff:ff:f1,netdev=mynet1,mrg_rxbuf=off,mq=on,vectors=10
Multi-queue is enabled
[root@localhost ~]# ethtool -l eth0
Channel parameters for eth0:
Pre-set maximums:
RX: 0
TX: 0
Other: 0
Combined: 4
Current hardware settings:
RX: 0
TX: 0
Other: 0
Combined: 4
Only one core is too busy，other cores are idle.
[root@localhost ~]# mpstat -P ALL 1
10:01:06 AM CPU %usr %nice %sys %iowait %irq %soft %steal
%guest %gnice %idle
10:01:07 AM all 0.26 0.00 8.85 0.00 0.00 12.50 3.65
0.00 0.00 74.74
10:01:07 AM 0 1.03 0.00 1.03 0.00 0.00 0.00 2.06
0.00 0.00 95.88
10:01:07 AM 1 0.00 0.00 0.00 0.00 0.00 0.00 0.00
0.00 0.00 100.00
10:01:07 AM 2 0.00 0.00 1.03 0.00 0.00 0.00 2.06
0.00 0.00 96.91
10:01:07 AM 3 1.09 0.00 35.87 0.00 0.00 52.17 10.87
0.00 0.00 0.00
[root@openstack06 mengyihua]# qperf -t 10 10.220.207.10 -ub -oo msg_size:1K
-vu tcp_bw
tcp_bw:
bw = 3.05 Gb/sec
msg_size = 1 KiB (1,024)
time = 10 sec
My physical card info
[root@openstack06 mengyihua]# ethtool eth4
Settings for eth4:
Supported ports: [ TP ]
Supported link modes: 100baseT/Full
1000baseT/Full
10000baseT/Full
Supported pause frame use: Symmetric
Supports auto-negotiation: Yes
Advertised link modes: 100baseT/Full
1000baseT/Full
10000baseT/Full
Advertised pause frame use: Symmetric
Advertised auto-negotiation: Yes
Speed: 10000Mb/s
Duplex: Full
Port: Twisted Pair
PHYAD: 0
Transceiver: external
Auto-negotiation: on
MDI-X: Unknown
Supports Wake-on: d
Wake-on: d
Current message level: 0x00000007 (7)
drv probe link
Best regards,Yihua Meng.
_______________________________________________
dev mailing list
[email protected]
https://mail.openvswitch.org/mailman/listinfo/ovs-dev
