Hi steven：
Thank you for your reply!
I followed your *advice(3)* , and made some attempts.
I create *three* startup config files of vpp:
The first one is named " *startup.conf.smp* ", the second one is named "
*startup.conf* "(my config file).
And The third one is named " *startup.conf.ok* ", it just delete "uio-driver
vfio-pci" on the basis of "startup.conf".
--------- *startup.conf.smp* ----------------
*unix { interactive }*
---------- *startup.conf* -------------------
unix {
nodaemon
log /var/log/vpp/vpp.log
full-coredump
cli-listen /run/vpp/cli.sock
gid vpp
}
api-trace { on }
api-segment { gid vpp }
socksvr { default }
cpu {
main-core 30
corelist-workers 26,28
workers 2
}
dpdk {
dev default {
num-rx-queues 1
num-tx-queues 2
}
dev 0000:3b:00.0
dev 0000:3b:00.1
#dev 0000:3b:00.2
#dev 0000:3b:00.3
*uio-driver vfio-pci*
}
---------- *startup.conf.ok* -------------------
unix {
nodaemon
log /var/log/vpp/vpp.log
full-coredump
cli-listen /run/vpp/cli.sock
gid vpp
}
api-trace { on }
api-segment { gid vpp }
socksvr { default }
cpu {
main-core 30
corelist-workers 26,28
workers 2
}
dpdk {
dev default {
num-rx-queues 1
num-tx-queues 2
}
dev 0000:3b:00.0
dev 0000:3b:00.1
#dev 0000:3b:00.2
#dev 0000:3b:00.3
*#uio-driver vfio-pci (just modify here on the basis of startup.conf--my config)
* *# @@@@@steven, do you know why this option makes the difference？@@@@@
*
*}*
-----------------------------------------------------------------------------
I'm not familiar with *testpmd* , but I will take some time to find out how it
works.
(1)When I use “ *startup.conf.smp* ", and follow the operation sequence below
after Centos startup, *it seems ok* :
[root@localhost ~]# *modprobe vfio-pci*
[root@localhost ~]# lsmod | grep vfio
vfio_pci 41412 2
vfio_iommu_type1 22440 0
vfio 32657 8 vfio_iommu_type1,vfio_pci
irqbypass 13503 4 kvm,vfio_pci
[root@localhost ~]# */usr/bin/numactl --cpubind=0 --membind=0 /usr/bin/vpp -c
/etc/vpp/startup.conf.smp*
...........
vpp# show pci
Address Sock VID:PID Link Speed Driver Product Name
Vital Product Data
...........
*0000:3b:00.0 0 8086:1572 8.0 GT/s x8 vfio-pci XL710 40GbE
Controller RV: 0x 86*
*0000:3b:00.1 0 8086:1572 8.0 GT/s x8 vfio-pci XL710 40GbE
Controller RV: 0x 86*
*0000:3b:00.2 0 8086:1572 8.0 GT/s x8 vfio-pci XL710 40GbE
Controller RV: 0x 86*
*0000:3b:00.3 0 8086:1572 8.0 GT/s x8 vfio-pci XL710 40GbE
Controller RV: 0x 86*
vpp# show interface
Name Idx State MTU (L3/IP4/IP6/MPLS) Counter
Count
*TenGigabitEthernet3b/0/0 1 down 9000/0/0/0*
*TenGigabitEthernet3b/0/1 2 down 9000/0/0/0*
*TenGigabitEthernet3b/0/2 3 down 9000/0/0/0*
*TenGigabitEthernet3b/0/3 4 down 9000/0/0/0*
local0 0 down 0/0/0/0
vpp# show log
2020/01/08 10:35:38:001 warn dpdk Unsupported PCI device
0x14e4:0x165f found at PCI address 0000:18:00.0
2020/01/08 10:35:38:017 warn dpdk Unsupported PCI device
0x14e4:0x165f found at PCI address 0000:18:00.1
2020/01/08 10:35:38:032 warn dpdk Unsupported PCI device
0x14e4:0x165f found at PCI address 0000:19:00.0
2020/01/08 10:35:38:076 warn dpdk Unsupported PCI device
0x14e4:0x165f found at PCI address 0000:19:00.1
2020/01/08 10:35:39:447 warn dpdk EAL init args: -c 2 -n 4
--in-memory --file-prefix vpp --master-lcore 1
2020/01/08 10:35:40:682 notice dpdk EAL: Detected 32 lcore(s)
2020/01/08 10:35:40:682 notice dpdk EAL: Detected 2 NUMA nodes
2020/01/08 10:35:40:682 notice dpdk EAL: Some devices want iova as va
but pa will be used because.. EAL: vfio-noiommu mode configured
2020/01/08 10:35:40:682 notice dpdk EAL: No available hugepages
reported in hugepages-1048576kB
2020/01/08 10:35:40:682 notice dpdk EAL: No free hugepages reported
in hugepages-1048576kB
2020/01/08 10:35:40:682 notice dpdk EAL: No free hugepages reported
in hugepages-1048576kB
2020/01/08 10:35:40:682 notice dpdk EAL: No available hugepages
reported in hugepages-1048576kB
2020/01/08 10:35:40:682 notice dpdk EAL: Probing VFIO support...
2020/01/08 10:35:40:682 notice dpdk EAL: VFIO support initialized
2020/01/08 10:35:40:682 notice dpdk EAL: WARNING! Base virtual
address hint (0xa80001000 != 0x7f4f40000000) not respected!
2020/01/08 10:35:40:682 notice dpdk EAL: This may cause issues
with mapping memory into secondary processes
*@Yichen, when I used "dmesg | grep Virtualization", nothing was returned:*
*[root@localhost ~]# dmesg | grep Virtualization*
*[root@localhost ~]#*
*I don't do performance test or other related tests now, so I don't know if
there are any other problems.*
(2)When I use *“startup.conf* ", and follow the operation sequence below after
Centos startup, *it seems bad* :
[root@localhost ~]# *modprobe vfio-pci*
[root@localhost ~]# lsmod | grep vfio
vfio_pci 41412 2
vfio_iommu_type1 22440 0
vfio 32657 8 vfio_iommu_type1,vfio_pci
irqbypass 13503 4 kvm,vfio_pci
[root@localhost ~]# */usr/bin/numactl --cpubind=0 --membind=0 /usr/bin/vpp -c
/etc/vpp/startup.conf*
...........
vpp# show pci
Address Sock VID:PID Link Speed Driver Product Name
Vital Product Data
...........
*0000:3b:00.0 0 8086:1572 8.0 GT/s x8 XL710 40GbE
Controller RV: 0x 86*
*0000:3b:00.1 0 8086:1572 8.0 GT/s x8 XL710 40GbE
Controller RV: 0x 86*
0000:3b:00.2 0 8086:1572 8.0 GT/s x8 i40e XL710 40GbE
Controller RV: 0x 86
0000:3b:00.3 0 8086:1572 8.0 GT/s x8 i40e XL710 40GbE
Controller RV: 0x 86
[root@localhost ~]# vppctl show int
Name Idx State MTU (L3/IP4/IP6/MPLS) Counter
Count
local0 0 down 0/0/0/0
vppctl show log
2020/01/08 11:01:40:400 warn dpdk EAL init args: -c 54000000 -n 4
--in-memory --file-prefix vpp -w 0000:3b:00.0 -w 0000:3b:00.1 --master-lcore 30
*2020/01/08 11:01:40:570 notice dpdk DPDK drivers found no ports...*
2020/01/08 11:01:40:575 notice dpdk EAL: Detected 32 lcore(s)
2020/01/08 11:01:40:575 notice dpdk EAL: Detected 2 NUMA nodes
2020/01/08 11:01:40:575 notice dpdk EAL: No available hugepages
reported in hugepages-1048576kB
2020/01/08 11:01:40:575 notice dpdk EAL: No free hugepages reported
in hugepages-1048576kB
2020/01/08 11:01:40:575 notice dpdk EAL: No free hugepages reported
in hugepages-1048576kB
2020/01/08 11:01:40:575 notice dpdk EAL: No available hugepages
reported in hugepages-1048576kB
2020/01/08 11:01:40:575 notice dpdk EAL: Probing VFIO support...
2020/01/08 11:01:40:575 notice dpdk EAL: VFIO support initialized
2020/01/08 11:01:40:575 notice dpdk EAL: WARNING! Base virtual
address hint (0xa80001000 != 0x7f0d40000000) not respected!
2020/01/08 11:01:40:575 notice dpdk EAL: This may cause issues
with mapping memory into secondary processes
[root@localhost ~]# *dmesg*
............
*[ 498.625440] VFIO - User Level meta-driver version: 0.3*
*[ 522.934678] i40e 0000:3b:00.0: removed PHC from p1p1*
*[ 523.142229] vfio-pci: probe of 0000:3b:00.0 failed with error -22*
*[ 523.154248] i40e 0000:3b:00.1: removed PHC from p1p2*
*[ 523.359757] vfio-pci: probe of 0000:3b:00.1 failed with error -22*
*[root@localhost ~]# dmesg | grep Virtualization*
*[root@localhost ~]#*
*@Yichen, here is same as above...*
(3) *I modify the "startup.conf" repeatedly, and finally found the dpdk option
"uio-driver vfio-pci" was a key point. if I delete it, the relust seems ok.*
[root@localhost ~]# modprobe vfio-pci
[root@localhost ~]# lsmod | grep vfio
vfio_pci 41412 2
vfio_iommu_type1 22440 0
vfio 32657 8 vfio_iommu_type1,vfio_pci
irqbypass 13503 4 kvm,vfio_pci
[root@localhost ~]# */usr/bin/numactl --cpubind=0 --membind=0 /usr/bin/vpp -c
/etc/vpp/startup.conf.ok*
...........
vpp# show pci
Address Sock VID:PID Link Speed Driver Product Name
Vital Product Data
...........
*0000:3b:00.0 0 8086:1572 8.0 GT/s x8 vfio-pci XL710 40GbE
Controller RV: 0x 86*
*0000:3b:00.1 0 8086:1572 8.0 GT/s x8 vfio-pci XL710 40GbE
Controller RV: 0x 86*
0000:3b:00.2 0 8086:1572 8.0 GT/s x8 i40e XL710 40GbE
Controller RV: 0x 86
0000:3b:00.3 0 8086:1572 8.0 GT/s x8 i40e XL710 40GbE
Controller RV: 0x 86
[root@localhost ~]# vppctl show int
Name Idx State MTU (L3/IP4/IP6/MPLS) Counter
Count
*TenGigabitEthernet3b/0/0 1 down 9000/0/0/0*
*TenGigabitEthernet3b/0/1 2 down 9000/0/0/0*
local0 0 down 0/0/0/0
[root@localhost ~]# vppctl show log
*2020/01/08 11:22:23:789 warn dpdk EAL init args: -c 54000000 -n 4
--in-memory --file-prefix vpp -w 0000:3b:00.0 -w 0000:3b:00.1 --master-lcore 30*
*2020/01/08 11:22:24:512 notice dpdk EAL: Detected 32 lcore(s)*
*2020/01/08 11:22:24:512 notice dpdk EAL: Detected 2 NUMA nodes*
2020/01/08 11:22:24:512 notice dpdk EAL: Some devices want iova as va
but pa will be used because.. EAL: vfio-noiommu mode configured
2020/01/08 11:22:24:512 notice dpdk EAL: No available hugepages
reported in hugepages-1048576kB
2020/01/08 11:22:24:512 notice dpdk EAL: No free hugepages reported
in hugepages-1048576kB
2020/01/08 11:22:24:512 notice dpdk EAL: No free hugepages reported
in hugepages-1048576kB
2020/01/08 11:22:24:512 notice dpdk EAL: No available hugepages
reported in hugepages-1048576kB
2020/01/08 11:22:24:512 notice dpdk EAL: Probing VFIO support...
2020/01/08 11:22:24:512 notice dpdk EAL: VFIO support initialized
2020/01/08 11:22:24:512 notice dpdk EAL: WARNING! Base virtual
address hint (0xa80001000 != 0x7fe540000000) not respected!
2020/01/08 11:22:24:512 notice dpdk EAL: This may cause issues
with mapping memory into secondary processes
[root@localhost dpdk]# *./dpdk-devbind --status*
Network devices using DPDK-compatible driver
============================================
*0000:3b:00.0 'Ethernet Controller X710 for 10GbE SFP+ 1572' drv=vfio-pci
unused=i40e*
*0000:3b:00.1 'Ethernet Controller X710 for 10GbE SFP+ 1572' drv=vfio-pci
unused=i40e*
Network devices using kernel driver
===================================
0000:18:00.0 'NetXtreme BCM5720 2-port Gigabit Ethernet PCIe 165f' if=em1
drv=tg3 unused=vfio-pci *Active*
0000:18:00.1 'NetXtreme BCM5720 2-port Gigabit Ethernet PCIe 165f' if=em2
drv=tg3 unused=vfio-pci
0000:19:00.0 'NetXtreme BCM5720 2-port Gigabit Ethernet PCIe 165f' if=em3
drv=tg3 unused=vfio-pci
0000:19:00.1 'NetXtreme BCM5720 2-port Gigabit Ethernet PCIe 165f' if=em4
drv=tg3 unused=vfio-pci *Active*
0000:3b:00.2 'Ethernet Controller X710 for 10GbE SFP+ 1572' if=p1p3 drv=i40e
unused=vfio-pci
0000:3b:00.3 'Ethernet Controller X710 for 10GbE SFP+ 1572' if=p1p4 drv=i40e
unused=vfio-pci
No 'Crypto' devices detected
============================
*root@localhost dpdk]# dmesg*
*..............*
*[ 167.814084] VFIO - User Level meta-driver version: 0.3*
*[ 270.848376] i40e 0000:3b:00.0: removed PHC from p1p1*
*[ 271.061138] iommu: Adding device 0000:3b:00.0 to group 0*
*[ 271.061159] vfio-pci 0000:3b:00.0: Adding kernel taint for vfio-noiommu
group on device*
*[ 271.073068] i40e 0000:3b:00.1: removed PHC from p1p2*
*[ 271.277631] iommu: Adding device 0000:3b:00.1 to group 1*
*[ 271.277652] vfio-pci 0000:3b:00.1: Adding kernel taint for vfio-noiommu
group on device*
*[ 271.458185] vfio-pci 0000:3b:00.0: Masking broken INTx support*
*[ 271.458313] vfio_ecap_init: 0000:3b:00.0 hiding ecap 0x19@0x1d0*
*[ 271.458326] vfio-pci 0000:3b:00.0: vfio-noiommu device opened by user
(vpp:2421)*
*[ 271.622873] vfio-pci 0000:3b:00.0: irq 74 for MSI/MSI-X*
*[ 271.725553] vfio-pci 0000:3b:00.1: Masking broken INTx support*
*[ 271.725684] vfio-pci 0000:3b:00.1: vfio-noiommu device opened by user
(vpp:2421)*
*[ 271.889681] vfio-pci 0000:3b:00.1: irq 75 for MSI/MSI-X*
*[root@localhost dpdk]# dmesg | grep Virtualization*
*[root@localhost dpdk]#*
*@Yichen, here is same as above...*
Regards，
gencli
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#15088): https://lists.fd.io/g/vpp-dev/message/15088
Mute This Topic: https://lists.fd.io/mt/69347948/21656
Mute #vpp: https://lists.fd.io/mk?hashtag=vpp&subid=1480452
Mute #vnet: https://lists.fd.io/mk?hashtag=vnet&subid=1480452
Group Owner: vpp-dev+ow...@lists.fd.io
Unsubscribe: https://lists.fd.io/g/vpp-dev/unsub [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
