On 23.5.2022. 10:41, Hrvoje Popovski wrote:
> On 23.5.2022. 8:34, Alexandr Nedvedicky wrote:
>> looks like kind of memory corruption. my bet is use-after-free.
>> will try to get to it later today.
>>
>> does it mean there is no such panic, when we handle IPv4 traffic only?
>
> Hi,
>
> yes, it seems that i can't trigger panic with ip4 only traffic, at least
> the same way i can with ip6 traffic
>
Here's another one but this time i've tcpdump outgoing ix interface.
I've tried same stuff with ip4 traffic and couldn't trigger panic.
10:53:59.682513 a192:a168:a100::111.9 > b192:b168:b111::bfbf.9: udp
puvamn_icf:au l t p(o0
oxflf_cfafcffhfe_fi82t2emf_62m6a8gi, c _ ch e c k : m b uf p l c p
u f r
0exe1 l7i, s t m o d if i e d : i t e m a d d r 0 x ff f f f
d8 0 a 37 f d a
0 0+ 1 6 0 xf f f ff d 8 0a 3 7 fd a f 2! = 0x c
0f1,8 9 2b)ec d f -5>9 b0 0 b
Stopped at db_enter+0x10: popq %rbp
TID PID UID PRFLAGS PFLAGS CPU COMMAND
32710 85256 0 0x14000 0x200 4K softnet
97437 83157 0 0x14000 0x200 1 softnet
212200 25091 0 0x14000 0x200 3 softnet
510395 50985 0 0x14000 0x200 5 softnet
417502 88838 0 0x14000 0x200 0 systq
db_enter() at db_enter+0x10
panic(ffffffff81f34fe0) at panic+0xbf
pool_cache_get(ffffffff82474c48) at pool_cache_get+0x25b
pool_get(ffffffff82474c48,2) at pool_get+0x61
m_clget(0,2,802) at m_clget+0xdd
ixgbe_get_buf(ffff8000000973a0,b2) at ixgbe_get_buf+0xa3
ixgbe_rxfill(ffff8000000973a0) at ixgbe_rxfill+0xaa
ixgbe_queue_intr(ffff800000024d00) at ixgbe_queue_intr+0x4f
intr_handler(ffff800022c89380,ffff800000081e00) at intr_handler+0x6e
Xintr_ioapic_edge0_untramp() at Xintr_ioapic_edge0_untramp+0x18f
acpicpu_idle() at acpicpu_idle+0x203
sched_idle(ffff800022412ff0) at sched_idle+0x280
end trace frame: 0x0, count: 3
https://www.openbsd.org/ddb.html describes the minimum info required in
bug reports. Insufficient info makes it difficult to find and fix bugs.
ddb{2}>
ddb{2}> show panic
cpu4: uvm_fault(0xffffffff822f6268, 0x17, 0, 2) -> e
*cpu2: pool_cache_item_magic_check: mbufpl cpu free list modified: item
addr 0x
fffffd80a37fda00+16 0xfffffd80a37fdaf2!=0xcf189becdf59b00b
ddb{2}>
ddb{2}> show reg
rdi 0
rsi 0x14
rbp 0xffff800022c88ff0
rbx 0xfffffd842f835c00
rdx 0xc800000000000000
rcx 0x206
rax 0x8a
r8 0x101010101010101
r9 0
r10 0xe6540fc793a8e615
r11 0x4860824aa7540a0c
r12 0xffff800022413a60
r13 0
r14 0
r15 0xffffffff81f34fe0 cmd0646_9_tim_udma+0x2acb1
rip 0xffffffff817b4d90 db_enter+0x10
cs 0x8
rflags 0x206
rsp 0xffff800022c88ff0
ss 0x10
db_enter+0x10: popq %rbp
ddb{2}> show mbuf
mbuf 0xffffffff817b4d90
m_type: -13108 m_flags:
c3cc<M_EOR,M_EXTWR,M_LOOP,M_BCAST,M_MCAST,M_COMP,M_LINK0>
m_next: 0x1d3b4c241c334c5d m_nextpkt: 0xcccc117400ae525c
m_data: 0xcccccccccccccccc m_len: 3435973836
m_dat: 0xffffffff817b4db0 m_pktdat: 0xffffffff817b4e00
ddb{2}> show all locks
Process 85256 (softnet) thread 0xffff8000ffffe7e0 (32710)
shared rwlock netlock r = 0 (0xffffffff822e9990)
shared rwlock softnet r = 0 (0xffff800000031370)
Process 83157 (softnet) thread 0xffff8000ffffea80 (97437)
shared rwlock netlock r = 0 (0xffffffff822e9990)
shared rwlock softnet r = 0 (0xffff800000031270)
Process 25091 (softnet) thread 0xffff8000ffffed20 (212200)
shared rwlock netlock r = 0 (0xffffffff822e9990)
shared rwlock softnet r = 0 (0xffff800000031170)
Process 50985 (softnet) thread 0xffff8000ffffefc0 (510395)
shared rwlock softnet r = 0 (0xffff800000031070)
Process 88838 (systq) thread 0xffff8000fffff500 (417502)
shared rwlock systq r = 0 (0xffffffff822eaf08)
Process 59744 (softclock) thread 0xffff8000fffff7a0 (200127)
exclusive kernel_lock &kernel_lock r = 0 (0xffffffff824b03c0)
shared rwlock timeout r = 0 (0xffffffff822b2fe8)
ddb{2}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
81137 105065 65725 76 3 0x100093 netio tcpdump
65725 227707 17816 76 3 0x1100093 ttyout tcpdump
17816 349982 1 0 3 0x10008b sigsusp ksh
96985 429538 1 0 3 0x100098 kqread cron
95498 144368 28860 95 3 0x1100092 kqread smtpd
43714 295842 28860 103 3 0x1100092 kqread smtpd
80683 116687 28860 95 3 0x1100092 kqread smtpd
35950 130878 28860 95 3 0x100092 kqread smtpd
27765 48615 28860 95 3 0x1100092 kqread smtpd
55438 323904 28860 95 3 0x1100092 kqread smtpd
28860 63495 1 0 3 0x100080 kqread smtpd
10757 429101 1 0 3 0x88 kqread sshd
87947 62304 1 0 3 0x100080 kqread ntpd
23988 365405 75537 83 3 0x100092 kqread ntpd
75537 417153 1 83 3 0x1100092 kqread ntpd
75937 221426 20564 73 3 0x1100090 kqread syslogd
20564 85971 1 0 3 0x100082 netio syslogd
86488 242884 0 0 3 0x14200 bored smr
2904 468564 0 0 3 0x14200 pgzero zerothread
91566 21036 0 0 3 0x14200 aiodoned aiodoned
27014 16901 0 0 3 0x14200 syncer update
77452 22391 0 0 3 0x14200 cleaner cleaner
40777 144356 0 0 3 0x14200 reaper reaper
29823 108484 0 0 3 0x14200 pgdaemon pagedaemon
54356 382343 0 0 3 0x14200 usbtsk usbtask
95950 238987 0 0 3 0x14200 usbatsk usbatsk
2104 521418 0 0 3 0x40014200 acpi0 acpi0
34300 118770 0 0 3 0x40014200 idle5
56665 162151 0 0 3 0x40014200 idle4
87384 503601 0 0 3 0x40014200 idle3
*99818 407037 0 0 7 0x40014200 idle2
93677 305415 0 0 3 0x40014200 idle1
29193 507372 0 0 3 0x14200 bored sensors
85256 32710 0 0 7 0x14200 softnet
83157 97437 0 0 7 0x14200 softnet
25091 212200 0 0 7 0x14200 softnet
50985 510395 0 0 7 0x14200 softnet
89231 30864 0 0 3 0x14200 bored systqmp
88838 417502 0 0 7 0x14200 systq
59744 200127 0 0 3 0x40014200 bored softclock
74972 241329 0 0 3 0x40014200 idle0
1 286126 0 0 3 0x82 wait init
0 0 -1 0 3 0x10200 scheduler swapper
ddb{2}> ps /o
TID PID UID PRFLAGS PFLAGS CPU COMMAND
32710 85256 0 0x14000 0x200 4K softnet
97437 83157 0 0x14000 0x200 1 softnet
212200 25091 0 0x14000 0x200 3 softnet
510395 50985 0 0x14000 0x200 5 softnet
417502 88838 0 0x14000 0x200 0 systq
ddb{2}> trace /t 0t32710
vport_if_enqueue(ffff800000747800,fffffd80a3037600) at vport_if_enqueue+0x19
veb_port_input(ffff800000095048,fffffd80a3037600,ecf4bbdaf7f8,ffff800001335200)
at veb_port_input+0x5b0
ether_input(ffff800000095048,fffffd80a3037600) at ether_input+0x100
if_input_process(ffff800000095048,ffff800022c779e8) at if_input_process+0x6f
ifiq_process(ffff800000099600) at ifiq_process+0x69
taskq_thread(ffff800000031300) at taskq_thread+0x11a
end trace frame: 0x0, count: -6
ddb{2}>
ddb{2}> trace /t 0t97437
vport_if_enqueue(ffff800000747800,fffffd80a4557d00) at vport_if_enqueue+0x19
veb_port_input(ffff800000095048,fffffd80a4557d00,ecf4bbdaf7f8,ffff800001335200)
at veb_port_input+0x5b0
ether_input(ffff800000095048,fffffd80a4557d00) at ether_input+0x100
if_input_process(ffff800000095048,ffff800022c718d8) at if_input_process+0x6f
ifiq_process(ffff800000099500) at ifiq_process+0x69
taskq_thread(ffff800000031200) at taskq_thread+0x11a
end trace frame: 0x0, count: -6
ddb{2}> trace /t 0t212200
vport_if_enqueue(ffff800000747800,fffffd80a30a4500) at vport_if_enqueue+0x19
veb_port_input(ffff800000095048,fffffd80a30a4500,ecf4bbdaf7f8,ffff800001335200)
at veb_port_input+0x5b0
ether_input(ffff800000095048,fffffd80a30a4500) at ether_input+0x100
if_input_process(ffff800000095048,ffff800022c6b088) at if_input_process+0x6f
ifiq_process(ffff800000099800) at ifiq_process+0x69
taskq_thread(ffff800000031100) at taskq_thread+0x11a
end trace frame: 0x0, count: -6
ddb{2}> trace /t 0t510395
ffff8000000b4048(dd86faf7dabbf4ec,894f1dbae2900000,e4ab0d00,b4b8b6295459002,55e
99bbe00001a00,900090036000030) at 0xffff8000000b4048
end trace frame: 0x0, count: -1
ddb{2}> trace /t 0t417502
sleep_finish(ffff800022c590d0,1) at sleep_finish+0xfe
rw_enter(ffffffff822e9980,1) at rw_enter+0x232
pf_purge(ffffffff824aa1d0) at pf_purge+0x34
taskq_thread(ffffffff822eae98) at taskq_thread+0x11a
end trace frame: 0x0, count: -4
ddb{2}>
ddb{2}> mach ddbcpu 0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff822deff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff824b01b8) at __mp_lock+0xaa
__mp_acquire_count(ffffffff824b01b8,1) at __mp_acquire_count+0x38
mi_switch() at mi_switch+0x299
sleep_finish(ffff800022c590d0,1) at sleep_finish+0xfe
rw_enter(ffffffff822e9980,1) at rw_enter+0x232
pf_purge(ffffffff824aa1d0) at pf_purge+0x34
taskq_thread(ffffffff822eae98) at taskq_thread+0x11a
end trace frame: 0x0, count: 5
ddb{0}> mach ddbcpu 1
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800022409ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff824b01b8) at __mp_lock+0xb3
ether_resolve(ffff800001369800,fffffd80a4557d00,ffff80000002f3c0,fffffd83b3c2ed
28,ffff800022c713f8) at ether_resolve+0x23b
ether_output(ffff800001369800,fffffd80a4557d00,ffff80000002f3c0,fffffd83b3c2ed2
8) at ether_output+0x2c
ip6_forward(fffffd80a4557d00,fffffd83b3c2ed28,0) at ip6_forward+0x5d1
ip6_input_if(ffff800022c716c8,ffff800022c716d4,29,0,ffff800000747800) at
ip6_input_if+0x80a
ipv6_input(ffff800000747800,fffffd80a4557d00) at ipv6_input+0x39
ether_input(ffff800000747800,fffffd80a4557d00) at ether_input+0x3ad
vport_if_enqueue(ffff800000747800,fffffd80a4557d00) at vport_if_enqueue+0x19
veb_port_input(ffff800000095048,fffffd80a4557d00,ecf4bbdaf7f8,ffff800001335200)
at veb_port_input+0x5b0
ether_input(ffff800000095048,fffffd80a4557d00) at ether_input+0x100
if_input_process(ffff800000095048,ffff800022c718d8) at if_input_process+0x6f
end trace frame: 0xffff800022c71920, count: 0
ddb{1}> mach ddbcpu 2
Stopped at db_enter+0x10: popq %rbp
db_enter() at db_enter+0x10
panic(ffffffff81f34fe0) at panic+0xbf
pool_cache_get(ffffffff82474c48) at pool_cache_get+0x25b
pool_get(ffffffff82474c48,2) at pool_get+0x61
m_clget(0,2,802) at m_clget+0xdd
ixgbe_get_buf(ffff8000000973a0,b2) at ixgbe_get_buf+0xa3
ixgbe_rxfill(ffff8000000973a0) at ixgbe_rxfill+0xaa
ixgbe_queue_intr(ffff800000024d00) at ixgbe_queue_intr+0x4f
intr_handler(ffff800022c89380,ffff800000081e00) at intr_handler+0x6e
Xintr_ioapic_edge0_untramp() at Xintr_ioapic_edge0_untramp+0x18f
acpicpu_idle() at acpicpu_idle+0x203
sched_idle(ffff800022412ff0) at sched_idle+0x280
end trace frame: 0x0, count: 3
ddb{2}> mach ddbcpu 3
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80002241bff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff824b01b8) at __mp_lock+0xa0
ether_resolve(ffff800001369800,fffffd80a30a4500,ffff80000002f3c0,fffffd83b3c2ed
28,ffff800022c6aba8) at ether_resolve+0x23b
ether_output(ffff800001369800,fffffd80a30a4500,ffff80000002f3c0,fffffd83b3c2ed2
8) at ether_output+0x2c
ip6_forward(fffffd80a30a4500,fffffd83b3c2ed28,0) at ip6_forward+0x5d1
ip6_input_if(ffff800022c6ae78,ffff800022c6ae84,29,0,ffff800000747800) at
ip6_input_if+0x80a
ipv6_input(ffff800000747800,fffffd80a30a4500) at ipv6_input+0x39
ether_input(ffff800000747800,fffffd80a30a4500) at ether_input+0x3ad
vport_if_enqueue(ffff800000747800,fffffd80a30a4500) at vport_if_enqueue+0x19
veb_port_input(ffff800000095048,fffffd80a30a4500,ecf4bbdaf7f8,ffff800001335200)
at veb_port_input+0x5b0
ether_input(ffff800000095048,fffffd80a30a4500) at ether_input+0x100
if_input_process(ffff800000095048,ffff800022c6b088) at if_input_process+0x6f
end trace frame: 0xffff800022c6b0d0, count: 0
ddb{3}> mach ddbcpu 4
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800022424ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
x86_bus_space_io_read_1(2f8,5) at x86_bus_space_io_read_1+0x15
comcnputc(801,65) at comcnputc+0x7f
cnputc(65) at cnputc+0x37
db_putchar(65) at db_putchar+0x2ea
kprintf() at kprintf+0x133b
db_printf(ffffffff81fe5b29) at db_printf+0x69
fault(ffffffff81f94a46) at fault+0x8e
kpageflttrap(ffff800022c77190,17) at kpageflttrap+0x190
kerntrap(ffff800022c77190) at kerntrap+0x91
alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b
pfsync_grab_snapshot(ffff800022c772a0,ffff80000073d000) at
pfsync_grab_snapshot+0xd0
end trace frame: 0xffff800022c77390, count: 0
ddb{4}> mach ddbcpu 5
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80002242dff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff824b01b8) at __mp_lock+0xb3
ether_resolve(ffff8000000b4048,fffffd80a3ef1600,ffff800022c65b68,0,ffff800022c6
5ae8) at ether_resolve+0x1ad
ether_output(ffff8000000b4048,fffffd80a3ef1600,ffff800022c65b68,0) at
ether_output+0x2c
ip_output(fffffd80a3ef1600,0,0,2,ffff80000073d870,0,772c0d4063fcfe3b) at
ip_output+0x8ee
pfsync_send_dispatch(ffffffff8241e1e8) at pfsync_send_dispatch+0xd2
taskq_thread(ffff800000031000) at taskq_thread+0x11a
end trace frame: 0x0, count: 6
ddb{5}>
