Hi all,
I can reproduce panic when sending ip6 traffic over vport and destroying
pfsync interface. It is reproducible with veb and vport but i couldn't
trigger panic when forwarding ip6 over physical interfaces.
I've compiled kernel with source fetched half an hour ago just to enable
WITNESS.
r620-1# ifconfig pfsync0 destroy
panicu:v m_ f a u lt ( 0 x f ff f f ff f 8 2 3 ba 6 1 8 , 0 x 17
, 0, 2 ) -
> e
pkoeronle_cla: c he _ i t em _ m a gi c _ ch e c k : m b u fp l
c pu f r ee
l i s t m o d if i e d : i t em a d dr 0
xpfagfef f fd 8 0 a 4 1 c3 f 0 0 +2 4 0 x a f5 5 1 e 6f 8 f 9 0
25 5 f != 0 x a f
55 1 e 6 f8 f 35 f d 5 f
fStopped at db_enter+0x10: popq %rbp
TID PID UID PRFLAGS PFLAGS CPU COMMAND
317552 39553 0 0x14000 0x200 2K softnet
504828 12606 0 0x14000 0x200 4 softnet
*283345 81494 0 0x14000 0x200 3 softnet
db_enter() at db_enter+0x10
panic(ffffffff81f39222) at panic+0xbf
pool_cache_get(ffffffff82323228) at pool_cache_get+0x25b
pool_get(ffffffff82323228,2) at pool_get+0x61
m_gethdr(2,1) at m_gethdr+0x3f
pfsync_sendout() at pfsync_sendout+0xe9
pfsync_update_state(fffffd839f1f8950) at pfsync_update_state+0x15b
pf_test(18,1,ffff800000095048,ffff800022c6ae30) at pf_test+0xd53
veb_pf(ffff800000095048,1,fffffd80a3594900) at veb_pf+0xbf
veb_port_input(ffff800000095048,fffffd80a3594900,ecf4bbdaf7f8,ffff800000747300)
at veb_port_input+0x2ce
ether_input(ffff800000095048,fffffd80a3594900) at ether_input+0x100
if_input_process(ffff800000095048,ffff800022c6afc8) at if_input_process+0x6f
ifiq_process(ffff800000099800) at ifiq_process+0x69
taskq_thread(ffff800000031100) at taskq_thread+0x11a
end trace frame: 0x0, count: 1
https://www.openbsd.org/ddb.html describes the minimum info required in
bug reports. Insufficient info makes it difficult to find and fix bugs.
ddb{3}> show panic
*cpu3: pool_cache_item_magic_check: mbufpl cpu free list modified: item
addr 0xfffffd80a41c3f00+24 0xaf551e6f8f90255f!=0xaf551e6f8f35fd5f
cpu2: uvm_fault(0xffffffff823ba618, 0x17, 0, 2) -> e
ddb{3}>
ddb{3}> show reg
rdi 0
rsi 0x14
rbp 0xffff800022c6a960
rbx 0xfffffd842f835c00
rdx 0xc800000000000000
rcx 0x282
rax 0x8a
r8 0x101010101010101
r9 0
r10 0xedcd3183c339b665
r11 0xb0f0eb58b1d2563
r12 0xffff80002241ca60
r13 0
r14 0
r15 0xffffffff81f39222 cmd0646_9_tim_udma+0x314d8
rip 0xffffffff8118e200 db_enter+0x10
cs 0x8
rflags 0x206
rsp 0xffff800022c6a960
ss 0x10
db_enter+0x10: popq %rbp
ddb{3}> show all locks
Process 39553 (softnet) thread 0xffff8000ffffed20 (317552)
shared rwlock netlock r = 0 (0xffffffff822c6550)
shared rwlock softnet r = 0 (0xffff800000031370)
Process 12606 (softnet) thread 0xffff8000ffffe000 (504828)
shared rwlock netlock r = 0 (0xffffffff822c6550)
shared rwlock softnet r = 0 (0xffff800000031270)
Process 81494 (softnet) thread 0xffff8000ffffe2a0 (283345)
shared rwlock netlock r = 0 (0xffffffff822c6550)
shared rwlock softnet r = 0 (0xffff800000031170)
Process 96881 (softnet) thread 0xffff8000ffffe540 (159803)
shared rwlock softnet r = 0 (0xffff800000031070)
Process 26865 (systq) thread 0xffff8000ffffea80 (449324)
shared rwlock systq r = 0 (0xffffffff822dd728)
Process 93339 (softclock) thread 0xffff8000ffffefc0 (160018)
shared rwlock timeout r = 0 (0xffffffff822b6000)
ddb{3}>
ddb{3}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
39455 263128 42512 0 3 0x3 netlock ifconfig
42512 258140 1 0 3 0x10008b sigsusp ksh
34696 282706 1 0 3 0x100098 kqread cron
86943 298932 81565 95 3 0x1100092 kqread smtpd
34037 448643 81565 103 3 0x1100092 kqread smtpd
17802 340759 81565 95 3 0x1100092 kqread smtpd
54979 438478 81565 95 3 0x100092 kqread smtpd
29724 438684 81565 95 3 0x1100092 kqread smtpd
3110 313509 81565 95 3 0x1100092 kqread smtpd
81565 137591 1 0 3 0x100080 kqread smtpd
81008 204817 1 0 3 0x88 kqread sshd
72442 275002 1 0 3 0x100080 kqread ntpd
97406 453489 91190 83 3 0x100092 kqread ntpd
91190 488051 1 83 3 0x1100012 netlock ntpd
31521 42595 4468 73 3 0x1100090 kqread syslogd
4468 43476 1 0 3 0x100082 netio syslogd
66713 499933 0 0 3 0x14200 bored smr
73608 203287 0 0 3 0x14200 pgzero zerothread
21951 139825 0 0 3 0x14200 aiodoned aiodoned
54722 61330 0 0 3 0x14200 syncer update
93833 423450 0 0 3 0x14200 cleaner cleaner
63772 65566 0 0 3 0x14200 reaper reaper
52126 91640 0 0 3 0x14200 pgdaemon pagedaemon
47046 419680 0 0 3 0x14200 usbtsk usbtask
31188 138481 0 0 3 0x14200 usbatsk usbatsk
52102 131157 0 0 3 0x40014200 acpi0 acpi0
81077 74193 0 0 7 0x40014200 idle5
53938 206103 0 0 3 0x40014200 idle4
18201 305025 0 0 3 0x40014200 idle3
373 241128 0 0 3 0x40014200 idle2
61581 113335 0 0 7 0x40014200 idle1
19442 218158 0 0 3 0x14200 bored sensors
39553 317552 0 0 7 0x14200 softnet
12606 504828 0 0 7 0x14200 softnet
*81494 283345 0 0 7 0x14200 softnet
96881 159803 0 0 3 0x14200 netlock softnet
57572 293943 0 0 3 0x14200 bored systqmp
26865 449324 0 0 3 0x14200 netlock systq
93339 160018 0 0 3 0x40014200 netlock softclock
69454 397018 0 0 7 0x40014200 idle0
1 44029 0 0 3 0x82 wait init
0 0 -1 0 3 0x10200 scheduler swapper
ddb{3}> ps /o
TID PID UID PRFLAGS PFLAGS CPU COMMAND
317552 39553 0 0x14000 0x200 2K softnet
504828 12606 0 0x14000 0x200 4 softnet
*283345 81494 0 0x14000 0x200 3 softnet
ddb{3}> trace /t 0t317552
vport_if_enqueue(ffff80000074b000,fffffd80a3f1de00) at vport_if_enqueue+0x19
veb_port_input(ffff800000095048,fffffd80a3f1de00,ecf4bbdaf7f8,ffff800000747300)
at veb_port_input+0x5b0
ether_input(ffff800000095048,fffffd80a3f1de00) at ether_input+0x100
if_input_process(ffff800000095048,ffff800022c77708) at if_input_process+0x6f
ifiq_process(ffff800000099600) at ifiq_process+0x69
taskq_thread(ffff800000031300) at taskq_thread+0x11a
end trace frame: 0x0, count: -6
ddb{3}> trace /t 0t504828
vport_if_enqueue(ffff80000074b000,fffffd80a4213b00) at vport_if_enqueue+0x19
veb_port_input(ffff800000095048,fffffd80a4213b00,ecf4bbdaf7f8,ffff800000747300)
at veb_port_input+0x5b0
ether_input(ffff800000095048,fffffd80a4213b00) at ether_input+0x100
if_input_process(ffff800000095048,ffff800022c70eb8) at if_input_process+0x6f
ifiq_process(ffff800000099900) at ifiq_process+0x69
taskq_thread(ffff800000031200) at taskq_thread+0x11a
end trace frame: 0x0, count: -6
ddb{3}> trace /t 0t283345
kernel: protection fault trap, code=0
Faulted in DDB; continuing...
ddb{3}>
ddb{3}> mach ddbcpu 0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff822c4ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff823d5580) at __mp_lock+0xa0
softintr_dispatch(0) at softintr_dispatch+0x49
Xsoftclock() at Xsoftclock+0x1f
acpicpu_idle() at acpicpu_idle+0x203
sched_idle(ffffffff822c4ff0) at sched_idle+0x280
end trace frame: 0x0, count: 7
ddb{0}> mach ddbcpu 1
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800022409ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x203
sched_idle(ffff800022409ff0) at sched_idle+0x280
end trace frame: 0x0, count: 10
ddb{1}> mach ddbcpu 2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800022412ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
x86_bus_space_io_write_1(2f8,0,66) at x86_bus_space_io_write_1+0x19
comcnputc(801,66) at comcnputc+0xcb
cnputc(66) at cnputc+0x37
db_putchar(66) at db_putchar+0x2ea
kprintf() at kprintf+0x133b
db_printf(ffffffff81f2d83d) at db_printf+0x69
db_ktrap(6,0,ffff800022c76eb0) at db_ktrap+0x196
kerntrap(ffff800022c76eb0) at kerntrap+0xa2
alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b
pfsync_grab_snapshot(ffff800022c76fc0,ffff80000073c000) at
pfsync_grab_snapshot+0xd0
pfsync_sendout() at pfsync_sendout+0x89
end trace frame: 0xffff800022c770f0, count: 0
ddb{2}> mach ddbcpu 3
Stopped at db_enter+0x10: popq %rbp
db_enter() at db_enter+0x10
panic(ffffffff81f39222) at panic+0xbf
pool_cache_get(ffffffff82323228) at pool_cache_get+0x25b
pool_get(ffffffff82323228,2) at pool_get+0x61
m_gethdr(2,1) at m_gethdr+0x3f
pfsync_sendout() at pfsync_sendout+0xe9
pfsync_update_state(fffffd839f1f8950) at pfsync_update_state+0x15b
pf_test(18,1,ffff800000095048,ffff800022c6ae30) at pf_test+0xd53
veb_pf(ffff800000095048,1,fffffd80a3594900) at veb_pf+0xbf
veb_port_input(ffff800000095048,fffffd80a3594900,ecf4bbdaf7f8,ffff800000747300)
at veb_port_input+0x2ce
ether_input(ffff800000095048,fffffd80a3594900) at ether_input+0x100
if_input_process(ffff800000095048,ffff800022c6afc8) at if_input_process+0x6f
ifiq_process(ffff800000099800) at ifiq_process+0x69
taskq_thread(ffff800000031100) at taskq_thread+0x11a
end trace frame: 0x0, count: 1
ddb{3}> mach ddbcpu 4
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800022424ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff823d5580) at __mp_lock+0xb3
ether_resolve(ffff80000074b800,fffffd80a4213b00,ffff80000095b2c0,fffffd842c2398
58,ffff800022c709d8) at ether_resolve+0x23b
ether_output(ffff80000074b800,fffffd80a4213b00,ffff80000095b2c0,fffffd842c23985
8) at ether_output+0x2c
ip6_forward(fffffd80a4213b00,fffffd842c239858,0) at ip6_forward+0x5d1
ip6_input_if(ffff800022c70ca8,ffff800022c70cb4,29,0,ffff80000074b000) at
ip6_input_if+0x80a
ipv6_input(ffff80000074b000,fffffd80a4213b00) at ipv6_input+0x39
ether_input(ffff80000074b000,fffffd80a4213b00) at ether_input+0x3ad
vport_if_enqueue(ffff80000074b000,fffffd80a4213b00) at vport_if_enqueue+0x19
veb_port_input(ffff800000095048,fffffd80a4213b00,ecf4bbdaf7f8,ffff800000747300)
at veb_port_input+0x5b0
ether_input(ffff800000095048,fffffd80a4213b00) at ether_input+0x100
if_input_process(ffff800000095048,ffff800022c70eb8) at if_input_process+0x6f
end trace frame: 0xffff800022c70f00, count: 0
ddb{4}> mach ddbcpu 5
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80002242dff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x203
sched_idle(ffff80002242dff0) at sched_idle+0x280
end trace frame: 0x0, count: 10
ddb{5}>
