On 10.5.2022. 22:55, Alexander Bluhm wrote:
> Yes. It is similar.
>
> I have read the whole mail thread and the final fix got commited.
> But it looks incomplete, pf is still sleeping.
>
> Hrvoje, can you run the tests again that triggered the panics a
> year ago?
Hi,
year ago panics was triggered when veb or tpmr bridged traffic. I've
tried that right now and I couldn't trigger that panics.
If I put vport and route traffic over veb than I can trigger panic with
or without vlans as child-iface for veb.
For panic i need to have pf enabled and to run
ifconfig veb destroy or ifconfig vlan destroy and sh netstart in loop.
this is panic without vlans
panic: kernel diagnostic assertion "curcpu()->ci_schedstate.spc_smrdepth
== 0" failed: file "/sys/kern/subr_xxx.c", line 163
Stopped at db_enter+0x10: popq %rbp
TID PID UID PRFLAGS PFLAGS CPU COMMAND
57981 52408 0 0x14000 0x200 3 softnet
18952 62179 0 0x14000 0x200 5 softnet
db_enter() at db_enter+0x10
panic(ffffffff81f36a34) at panic+0xbf
__assert(ffffffff81faa7fa,ffffffff81fd47a7,a3,ffffffff81fe7c9d) at
__assert+0x25
assertwaitok() at assertwaitok+0xcc
mi_switch() at mi_switch+0x40
sleep_finish(ffff800022c707a0,1) at sleep_finish+0x10b
rw_enter(ffffffff822b3ad8,2) at rw_enter+0x232
pf_test(2,3,ffff8000000c6048,ffff800022c70a58) at pf_test+0xcf0
ip_output(fffffd80a32f1f00,0,ffff800022c70be8,1,0,0,e8e0f1a7c10273fe) at
ip_output+0x6b7
ip_forward(fffffd80a32f1f00,ffff8000014ee000,fffffd83a8657078,0) at
ip_forward+0x2da
ip_input_if(ffff800022c70d28,ffff800022c70d34,4,0,ffff8000014ee000) at
ip_input_if+0x353
ipv4_input(ffff8000014ee000,fffffd80a32f1f00) at ipv4_input+0x39
ether_input(ffff8000014ee000,fffffd80a32f1f00) at ether_input+0x3ad
vport_if_enqueue(ffff8000014ee000,fffffd80a32f1f00) at vport_if_enqueue+0x19
end trace frame: 0xffff800022c70e70, count: 0
https://www.openbsd.org/ddb.html describes the minimum info required in
bug reports. Insufficient info makes it difficult to find and fix bugs.
ddb{4}>
ddb{4}> show reg
rdi 0
rsi 0x14
rbp 0xffff800022c705f0
rbx 0xffff800022424ff0
rdx 0x8000000000000000
rcx 0x286
rax 0x7d
r8 0x101010101010101
r9 0
r10 0x5b4ef42a9c796b43
r11 0xada7e964a691819f
r12 0xffff800022425a60
r13 0xffff800022c450a0
r14 0
r15 0xffffffff81f36a34 cmd0646_9_tim_udma+0x2d9d2
rip 0xffffffff81c01c30 db_enter+0x10
cs 0x8
rflags 0x286
rsp 0xffff800022c705f0
ss 0x10
db_enter+0x10: popq %rbp
ddb{4}>
ddb{4}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
14129 480066 92457 0 3 0x3 netlock ifconfig
92457 504149 2002 0 3 0x10008b sigsusp sh
2002 26492 1517 0 3 0x10008b sigsusp sh
1517 131574 1 0 3 0x10008b sigsusp ksh
26252 251094 1 0 3 0x100098 kqread cron
20251 457205 97875 95 3 0x1100092 kqread smtpd
62139 255853 97875 103 3 0x1100092 kqread smtpd
29505 64154 97875 95 3 0x1100092 kqread smtpd
20035 471489 97875 95 3 0x100092 kqread smtpd
91114 73268 97875 95 3 0x1100092 kqread smtpd
78396 414422 97875 95 3 0x1100092 kqread smtpd
97875 113010 1 0 3 0x100080 kqread smtpd
21916 226987 1 0 3 0x88 kqread sshd
9017 4247 1 0 3 0x100080 kqread ntpd
72358 391459 38133 83 3 0x100092 kqread ntpd
38133 355054 1 83 3 0x1100012 netlock ntpd
91824 285625 60194 73 3 0x1100090 kqread syslogd
60194 367623 1 0 3 0x100082 netio syslogd
73270 113983 0 0 3 0x14200 bored smr
51379 478537 0 0 3 0x14200 pgzero zerothread
85386 54454 0 0 3 0x14200 aiodoned aiodoned
10937 491268 0 0 3 0x14200 syncer update
85008 360847 0 0 3 0x14200 cleaner cleaner
76642 501363 0 0 3 0x14200 reaper reaper
32934 257878 0 0 3 0x14200 pgdaemon pagedaemon
48583 371156 0 0 3 0x14200 usbtsk usbtask
53660 310701 0 0 3 0x14200 usbatsk usbatsk
19211 31258 0 0 3 0x40014200 acpi0 acpi0
11856 305318 0 0 3 0x40014200 idle5
9933 290633 0 0 3 0x40014200 idle4
41570 94891 0 0 3 0x40014200 idle3
8005 288623 0 0 7 0x40014200 idle2
94335 428797 0 0 7 0x40014200 idle1
41936 315516 0 0 3 0x14200 bored sensors
29925 368584 0 0 3 0x14200 pf_state_loc softnet
*57081 250665 0 0 3 0x14200 pf_state_loc softnet
52408 57981 0 0 7 0x14200 softnet
62179 18952 0 0 7 0x14200 softnet
27530 318849 0 0 3 0x14200 bored systqmp
85837 522405 0 0 3 0x14200 netlock systq
77716 326935 0 0 3 0x40014200 netlock softclock
7796 40566 0 0 7 0x40014200 idle0
1 473596 0 0 3 0x82 wait init
0 0 -1 0 3 0x10200 scheduler swapper
ddb{4}> show all locks
CPU 4:
exclusive sched_lock &sched_lock r = 0 (0xffffffff824819c0)
#0 witness_lock+0x311
#1 sleep_setup+0xa5
#2 rw_enter+0x211
#3 pf_test+0xcf0
#4 ip_output+0x6b7
#5 ip_forward+0x2da
#6 ip_input_if+0x353
#7 ipv4_input+0x39
#8 ether_input+0x3ad
#9 vport_if_enqueue+0x19
#10 veb_port_input+0x529
#11 ether_input+0x100
#12 if_input_process+0x6f
#13 ifiq_process+0x69
#14 taskq_thread+0x11a
#15 proc_trampoline+0x1c
Process 29925 (softnet) thread 0xffff8000ffffefc0 (368584)
shared rwlock netlock r = 0 (0xffffffff822c3538)
#0 witness_lock+0x311
#1 if_input_process+0x43
#2 ifiq_process+0x69
#3 taskq_thread+0x11a
#4 proc_trampoline+0x1c
shared rwlock softnet r = 0 (0xffff800000031370)
#0 witness_lock+0x311
#1 taskq_thread+0x10d
#2 proc_trampoline+0x1c
Process 57081 (softnet) thread 0xffff8000ffffe7e0 (250665)
shared rwlock netlock r = 0 (0xffffffff822c3538)
#0 witness_lock+0x311
#1 rw_enter+0x292
#2 if_input_process+0x43
#3 ifiq_process+0x69
#4 taskq_thread+0x11a
#5 proc_trampoline+0x1c
shared rwlock softnet r = 0 (0xffff800000031270)
#0 witness_lock+0x311
#1 taskq_thread+0x10d
#2 proc_trampoline+0x1c
exclusive sched_lock &sched_lock r = 0 (0xffffffff824819c0)
#0 witness_lock+0x311
#1 sleep_setup+0xa5
#2 rw_enter+0x211
#3 pf_test+0xcf0
#4 ip_output+0x6b7
#5 ip_forward+0x2da
#6 ip_input_if+0x353
#7 ipv4_input+0x39
#8 ether_input+0x3ad
#9 vport_if_enqueue+0x19
#10 veb_port_input+0x529
#11 ether_input+0x100
#12 if_input_process+0x6f
#13 ifiq_process+0x69
#14 taskq_thread+0x11a
#15 proc_trampoline+0x1c
Process 52408 (softnet) thread 0xffff8000ffffe000 (57981)
shared rwlock netlock r = 0 (0xffffffff822c3538)
#0 witness_lock+0x311
#1 if_input_process+0x43
#2 ifiq_process+0x69
#3 taskq_thread+0x11a
#4 proc_trampoline+0x1c
shared rwlock softnet r = 0 (0xffff800000031170)
#0 witness_lock+0x311
#1 taskq_thread+0x10d
#2 proc_trampoline+0x1c
Process 62179 (softnet) thread 0xffff8000ffffe2a0 (18952)
exclusive rwlock pf_state_lock r = 0 (0xffffffff822b3ae8)
#0 witness_lock+0x311
#1 rw_enter+0x292
#2 pf_state_insert+0x7b
#3 pf_test_rule+0xe8e
#4 pf_test+0x10a1
#5 veb_pf+0xbf
#6 veb_port_input+0x44a
#7 ether_input+0x100
#8 if_input_process+0x6f
#9 ifiq_process+0x69
#10 taskq_thread+0x11a
#11 proc_trampoline+0x1c
exclusive rwlock pf_lock r = 0 (0xffffffff822b3ab8)
#0 witness_lock+0x311
#1 pf_test+0x107a
#2 veb_pf+0xbf
#3 veb_port_input+0x44a
#4 ether_input+0x100
#5 if_input_process+0x6f
#6 ifiq_process+0x69
#7 taskq_thread+0x11a
#8 proc_trampoline+0x1c
shared rwlock netlock r = 0 (0xffffffff822c3538)
#0 witness_lock+0x311
#1 rw_enter+0x292
#2 if_input_process+0x43
#3 ifiq_process+0x69
#4 taskq_thread+0x11a
#5 proc_trampoline+0x1c
shared rwlock softnet r = 0 (0xffff800000031070)
#0 witness_lock+0x311
#1 taskq_thread+0x10d
#2 proc_trampoline+0x1c
Process 85837 (systq) thread 0xffff8000ffffea80 (522405)
shared rwlock systq r = 0 (0xffffffff822b4590)
#0 witness_lock+0x311
#1 taskq_thread+0x10d
#2 proc_trampoline+0x1c
Process 77716 (softclock) thread 0xffff8000ffffed20 (326935)
shared rwlock timeout r = 0 (0xffffffff822b2e00)
#0 witness_lock+0x311
#1 timeout_run+0x88
#2 softclock_thread+0x11d
#3 proc_trampoline+0x1c
ddb{4}>
ddb{4}> mach ddbcpu 0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff822a7ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff824817b8) at __mp_lock+0xaa
wakeup_n(ffffffff823381a0,ffffffff) at wakeup_n+0x32
softintr_dispatch(0) at softintr_dispatch+0xeb
Xsoftclock() at Xsoftclock+0x1f
acpicpu_idle() at acpicpu_idle+0x203
sched_idle(ffffffff822a7ff0) at sched_idle+0x280
end trace frame: 0x0, count: 6
ddb{0}> mach ddbcpu 1
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800022409ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x203
sched_idle(ffff800022409ff0) at sched_idle+0x280
end trace frame: 0x0, count: 10
ddb{1}> mach ddbcpu 2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800022412ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff824817b8) at __mp_lock+0xb3
wakeup_n(ffff800000031100,1) at wakeup_n+0x32
task_add(ffff800000031100,ffff800000099860) at task_add+0x7f
ifiq_input(ffff800000099800,ffff800022c88e70) at ifiq_input+0x188
ixgbe_rxeof(ffff8000000973a0) at ixgbe_rxeof+0x32b
ixgbe_queue_intr(ffff800000024d00) at ixgbe_queue_intr+0x3c
intr_handler(ffff800022c88fb0,ffff800000081e00) at intr_handler+0x6e
Xintr_ioapic_edge0_untramp() at Xintr_ioapic_edge0_untramp+0x18f
acpicpu_idle() at acpicpu_idle+0x203
sched_idle(ffff800022412ff0) at sched_idle+0x280
end trace frame: 0x0, count: 2
ddb{2}> mach ddbcpu 3
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80002241bff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff824817b8) at __mp_lock+0xb3
wakeup_n(ffffffff822b3ad8,ffffffff) at wakeup_n+0x32
pf_test(2,1,ffff800000095048,ffff800022c6b320) at pf_test+0xd31
veb_pf(ffff800000095048,1,fffffd80a3310c00) at veb_pf+0xbf
veb_port_input(ffff800000095048,fffffd80a3310c00,ecf4bbdaf7f8,ffff8000014c3400)
at veb_port_input+0x2e8
ether_input(ffff800000095048,fffffd80a3310c00) at ether_input+0x100
if_input_process(ffff800000095048,ffff800022c6b4b8) at if_input_process+0x6f
ifiq_process(ffff800000099800) at ifiq_process+0x69
taskq_thread(ffff800000031100) at taskq_thread+0x11a
end trace frame: 0x0, count: 3
ddb{3}> mach ddbcpu 4
Stopped at db_enter+0x10: popq %rbp
db_enter() at db_enter+0x10
panic(ffffffff81f36a34) at panic+0xbf
__assert(ffffffff81faa7fa,ffffffff81fd47a7,a3,ffffffff81fe7c9d) at
__assert+0x25
assertwaitok() at assertwaitok+0xcc
mi_switch() at mi_switch+0x40
sleep_finish(ffff800022c707a0,1) at sleep_finish+0x10b
rw_enter(ffffffff822b3ad8,2) at rw_enter+0x232
pf_test(2,3,ffff8000000c6048,ffff800022c70a58) at pf_test+0xcf0
ip_output(fffffd80a32f1f00,0,ffff800022c70be8,1,0,0,e8e0f1a7c10273fe) at
ip_output+0x6b7
ip_forward(fffffd80a32f1f00,ffff8000014ee000,fffffd83a8657078,0) at
ip_forward+0x2da
ip_input_if(ffff800022c70d28,ffff800022c70d34,4,0,ffff8000014ee000) at
ip_input_if+0x353
ipv4_input(ffff8000014ee000,fffffd80a32f1f00) at ipv4_input+0x39
ether_input(ffff8000014ee000,fffffd80a32f1f00) at ether_input+0x3ad
vport_if_enqueue(ffff8000014ee000,fffffd80a32f1f00) at vport_if_enqueue+0x19
end trace frame: 0xffff800022c70e70, count: 0
ddb{4}> mach ddbcpu 5
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80002242dff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff824817b8) at __mp_lock+0xa0
wakeup_n(ffff800000031000,1) at wakeup_n+0x32
task_add(ffff800000031000,ffff8000000c63e8) at task_add+0x7f
if_enqueue_ifq(ffff8000000c6048,fffffd80a379e300) at if_enqueue_ifq+0x6b
ether_output(ffff8000000c6048,fffffd80a379e300,ffff8000014e9e60,fffffd83a865707
8) at ether_output+0x7e
ip_output(fffffd80a379e300,0,ffff800022c655e8,1,0,0,e8e0f1a7c10273fe) at
ip_output+0x8ee
ip_forward(fffffd80a379e300,ffff8000014ee000,fffffd83a8657078,0) at
ip_forward+0x2da
ip_input_if(ffff800022c65728,ffff800022c65734,4,0,ffff8000014ee000) at
ip_input_if+0x353
ipv4_input(ffff8000014ee000,fffffd80a379e300) at ipv4_input+0x39
ether_input(ffff8000014ee000,fffffd80a379e300) at ether_input+0x3ad
vport_if_enqueue(ffff8000014ee000,fffffd80a379e300) at vport_if_enqueue+0x19
end trace frame: 0xffff800022c65870, count: 0
ddb{5}>
ix0: flags=8b43<UP,BROADCAST,RUNNING,PROMISC,ALLMULTI,SIMPLEX,MULTICAST>
mtu 1500
lladdr ec:f4:bb:da:f7:f8
index 1 priority 0 llprio 3
media: Ethernet autoselect (10GSFP+Cu full-duplex,rxpause,txpause)
status: active
ix1: flags=8b43<UP,BROADCAST,RUNNING,PROMISC,ALLMULTI,SIMPLEX,MULTICAST>
mtu 1500
lladdr ec:f4:bb:da:f7:fa
index 2 priority 0 llprio 3
media: Ethernet autoselect (10GSFP+Cu full-duplex,rxpause,txpause)
status: active
veb0: flags=a843<UP,BROADCAST,RUNNING,SIMPLEX,LINK1,MULTICAST>
index 9 llprio 3
groups: veb
ix0 flags=3<LEARNING,DISCOVER>
port 1 ifpriority 0 ifcost 0
vport0 flags=3<LEARNING,DISCOVER>
port 11 ifpriority 0 ifcost 0
veb1: flags=a843<UP,BROADCAST,RUNNING,SIMPLEX,LINK1,MULTICAST>
index 10 llprio 3
groups: veb
ix1 flags=3<LEARNING,DISCOVER>
port 2 ifpriority 0 ifcost 0
vport1 flags=3<LEARNING,DISCOVER>
port 12 ifpriority 0 ifcost 0
vport0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> mtu 1500
lladdr ec:f4:bb:da:f7:f8
index 11 priority 0 llprio 3
groups: vport
inet 192.168.100.11 netmask 0xffffff00 broadcast 192.168.100.255
vport1: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> mtu 1500
lladdr ec:f4:bb:da:f7:fa
index 12 priority 0 llprio 3
groups: vport
inet 192.168.111.11 netmask 0xffffff00 broadcast 192.168.111.255
r620-1# netstat -rnf inet | grep vport
16/8 192.168.100.111 UGS 0 0 - 8
vport0
48/8 192.168.111.111 UGS 0 0 - 8
vport1
192.168.100/24 192.168.100.11 UCn 1 0 - 4
vport0
192.168.100.11 ec:f4:bb:da:f7:f8 UHLl 0 0 - 1
vport0
192.168.100.111 link#11 UHLch 1 2 - 3
vport0
192.168.100.255 192.168.100.11 UHb 0 0 - 1
vport0
192.168.111/24 192.168.111.11 UCn 1 0 - 4
vport1
192.168.111.11 ec:f4:bb:da:f7:fa UHLl 0 0 - 1
vport1
192.168.111.111 link#12 UHLch 1 2 - 3
vport1
192.168.111.255 192.168.111.11 UHb 0 0 - 1
vport1
