On 22.4.2021. 1:10, Hrvoje Popovski wrote:
> On 22.4.2021. 0:31, Alexandr Nedvedicky wrote:
>> Hello,
>>
>> </snip>
>>>> Hi,
>>>>
>>>> with this diff i'm getting panic when i'm pushing traffic over that box.
>>>> This is plain forwarding. To compile with witness ?
>>>
>>>
>>> with witness
>>>
>>
>> any chance to check other CPUs to see what code they are executing?
>> I hope to be lucky enough and see thread, which could corrupt the memory.
>
>
> no problem, will do that tomorrow
>
something like this:
x3550m4# pappnaiannc:iicc :p:o ppoolo_oolcla__ddcohoe__gg_eiettt::e m
_mmcbmualg2fkpilc2_:: chppeaag
gceke: ee mmmbppttuyfyp
l Stopped at db_enter+0x10: popq %rbp
TID PID UID PRFLAGS PFLAGS CPU COMMAND
110314 50947 0 0x14000 0x200 2 softnet
370908 43261 0 0x14000 0x200 3 softnet
271599 79025 0 0x14000 0x200 0 softnet
121483 71835 0 0x14000 0x200 1 softnet
db_enter() at db_enter+0x10
panic(ffffffff81df855b) at panic+0x12a
pool_do_get(ffffffff8221e568,2,ffff800026e43294) at pool_do_get+0x309
pool_get(ffffffff8221e568,2) at pool_get+0x95
m_clget(0,2,802) at m_clget+0xdd
ixgbe_get_buf(ffff80000015c0e8,e) at ixgbe_get_buf+0xa3
ixgbe_rxfill(ffff80000015c0e8) at ixgbe_rxfill+0xae
ixgbe_queue_intr(ffff80000011ac40) at ixgbe_queue_intr+0x4f
intr_handler(ffff800026e434b0,ffff8000000cd700) at intr_handler+0x6e
Xintr_ioapic_edge4_untramp() at Xintr_ioapic_edge4_untramp+0x18f
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff800021a33ff0) at sched_idle+0x27e
end trace frame: 0x0, count: 3
https://www.openbsd.org/ddb.html describes the minimum info required in bug
reports. Insufficient info makes it difficult to find and fix bugs.
ddb{11}>
ddb{11}> show panic
pool_cache_item_magic_check: mbufpl cpu free list modified: item addr
0xfffffd8
066b54500+16 0xfffffd80792f4bbe!=0xb49ab28a8a25d432
ddb{11}> trace
db_enter() at db_enter+0x10
panic(ffffffff81df855b) at panic+0x12a
pool_do_get(ffffffff8221e568,2,ffff800026e43294) at pool_do_get+0x309
pool_get(ffffffff8221e568,2) at pool_get+0x95
m_clget(0,2,802) at m_clget+0xdd
ixgbe_get_buf(ffff80000015c0e8,e) at ixgbe_get_buf+0xa3
ixgbe_rxfill(ffff80000015c0e8) at ixgbe_rxfill+0xae
ixgbe_queue_intr(ffff80000011ac40) at ixgbe_queue_intr+0x4f
intr_handler(ffff800026e434b0,ffff8000000cd700) at intr_handler+0x6e
Xintr_ioapic_edge4_untramp() at Xintr_ioapic_edge4_untramp+0x18f
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff800021a33ff0) at sched_idle+0x27e
end trace frame: 0x0, count: -12
ddb{11}> show all locks
Process 59777 (ld) thread 0xffff800026ea97a8 (7337)
exclusive rrwlock inode r = 0 (0xfffffd887e57bc58)
#0 witness_lock+0x339
#1 rw_enter+0x286
#2 rrw_enter+0x56
#3 VOP_LOCK+0x33
#4 vn_lock+0x84
#5 uvn_get+0x15f
#6 uvm_fault_lower+0x2ae
#7 uvm_fault+0x19e
#8 upageflttrap+0x69
#9 usertrap+0x18b
#10 recall_trap+0x8
Process 50947 (softnet) thread 0xffff800026dc8a80 (110314)
shared rwlock netlock r = 0 (0xffffffff820e0ea8)
#0 witness_lock+0x339
#1 if_input_process+0x43
#2 ifiq_process+0x69
#3 taskq_thread+0x9f
#4 proc_trampoline+0x1c
shared rwlock softnet r = 0 (0xffff800000030370)
#0 witness_lock+0x339
#1 taskq_thread+0x92
#2 proc_trampoline+0x1c
ddb{11}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
59777 7337 5166 0 3 0x2 biowait ld
5166 135990 73189 0 3 0x10008a sigsusp sh
73189 29786 50754 0 3 0x10008a sigsusp make
1912 5584 1 0 3 0x100083 ttyin ksh
50754 423311 1 0 3 0x10008b sigsusp ksh
41817 523177 1 0 3 0x100098 poll cron
55124 161629 41428 720 3 0x90 kqread lldpd
41428 491873 1 0 3 0x80 netio lldpd
1877 283481 6199 95 3 0x100092 kqread smtpd
65700 262335 6199 103 3 0x100092 kqread smtpd
7456 289409 6199 95 3 0x100092 kqread smtpd
4455 333406 6199 95 3 0x100092 kqread smtpd
11921 364120 6199 95 3 0x100092 kqread smtpd
28535 449450 6199 95 3 0x100092 kqread smtpd
6199 140680 1 0 3 0x100080 kqread smtpd
69702 89036 1 0 3 0x80 select sshd
98630 388346 1 0 3 0x100080 poll ntpd
65290 297654 68128 83 3 0x100092 poll ntpd
68128 253220 1 83 3 0x100092 poll ntpd
8458 234468 4179 73 3 0x100090 kqread syslogd
4179 69049 1 0 3 0x100082 netio syslogd
25880 194929 0 0 3 0x14200 bored smr
22006 336506 0 0 3 0x14200 pgzero zerothread
26528 405700 0 0 3 0x14200 aiodoned aiodoned
9962 288230 0 0 3 0x14200 syncer update
77445 330499 0 0 3 0x14200 cleaner cleaner
63206 131307 0 0 3 0x14200 reaper reaper
96918 414726 0 0 3 0x14200 pgdaemon pagedaemon
45138 302760 0 0 3 0x14200 bored crynlk
35173 254614 0 0 3 0x14200 bored crypto
38927 388912 0 0 3 0x14200 usbtsk usbtask
40932 443213 0 0 3 0x14200 usbatsk usbatsk
62690 236765 0 0 3 0x40014200 acpi0 acpi0
*12785 154035 0 0 7 0x40014200 idle11
92814 287737 0 0 7 0x40014200 idle10
89941 345526 0 0 7 0x40014200 idle9
4551 127256 0 0 7 0x40014200 idle8
46539 340118 0 0 7 0x40014200 idle7
22570 403894 0 0 7 0x40014200 idle6
25137 289937 0 0 7 0x40014200 idle5
35027 21975 0 0 7 0x40014200 idle4
36397 159078 0 0 3 0x40014200 idle3
86589 412583 0 0 3 0x40014200 idle2
28344 397150 0 0 3 0x40014200 idle1
4270 497574 0 0 3 0x14200 bored sensors
50947 110314 0 0 7 0x14200 softnet
43261 370908 0 0 7 0x14200 softnet
79025 271599 0 0 7 0x14200 softnet
71835 121483 0 0 7 0x14200 softnet
30900 328163 0 0 3 0x14200 bored systqmp
98411 74459 0 0 3 0x14200 bored systq
54153 458634 0 0 3 0x40014200 bored softclock
38268 121628 0 0 3 0x40014200 idle0
1 76280 0 0 3 0x82 wait init
0 0 -1 0 3 0x10200 scheduler swapper
ddb{11}> trace /t 0t154035
sched_idle(ffff800021a33ff0) at sched_idle+0x27e
end trace frame: 0x0, count: -1
ddb{11}> mach ddbcpu 0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff820f0ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
mtx_enter(ffff800000030118) at mtx_enter+0x3c
msleep(ffff800000030100,ffff800000030118,20,ffffffff81e23340,0) at
msleep+0x135
taskq_next_work(ffff800000030100,ffff800026ded5b8) at taskq_next_work+0x61
taskq_thread(ffff800000030100) at taskq_thread+0xce
end trace frame: 0x0, count: 8
ddb{0}> mach ddbcpu 1
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000219d9ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff82227be0) at __mp_lock+0xa7
if_netisr(0) at if_netisr+0xc5
taskq_thread(ffff800000030000) at taskq_thread+0x9f
end trace frame: 0x0, count: 9
ddb{1}> mach ddbcpu 2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000219e2ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_release_all(ffffffff82227be0) at __mp_release_all+0x7a
msleep(ffff800000030300,ffff800000030318,20,ffffffff81e23340,0) at
msleep+0x116
taskq_next_work(ffff800000030300,ffff800026df9908) at taskq_next_work+0x61
taskq_thread(ffff800000030300) at taskq_thread+0xce
end trace frame: 0x0, count: 8
ddb{2}> mach ddbcpu 3
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000219ebff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
msleep(ffff800000030200,ffff800000030218,20,ffffffff81e23340,0) at
msleep+0x1c
taskq_next_work(ffff800000030200,ffff800026df3cf8) at taskq_next_work+0x61
taskq_thread(ffff800000030200) at taskq_thread+0xce
end trace frame: 0x0, count: 9
ddb{3}> mach ddbcpu 4
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000219f4ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff8000219f4ff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{4}> mach ddbcpu 5
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000219fdff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff8000219fdff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{5}> mach ddbcpu 6
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800021a06ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
x86_bus_space_io_read_1(3f8,5) at x86_bus_space_io_read_1+0x15
comcnputc(800,20) at comcnputc+0x12f
cnputc(20) at cnputc+0x37
kputchar(20,5,0) at kputchar+0x10e
kprintf() at kprintf+0x7de
panic(ffffffff81df858d) at panic+0xe6
pool_cache_get(ffffffff8221e568) at pool_cache_get+0x25b
pool_get(ffffffff8221e568,2) at pool_get+0x5e
m_clget(0,2,802) at m_clget+0xdd
ixgbe_get_buf(ffff80000015c570,14) at ixgbe_get_buf+0xa3
ixgbe_rxfill(ffff80000015c570) at ixgbe_rxfill+0x93
end trace frame: 0xffff800026e25ce0, count: 0
ddb{6}> mach ddbcpu 7
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800021a0fff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff800021a0fff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{7}> mach ddbcpu 8
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800021a18ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff800021a18ff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{9}> trace
x86_ipi_db(ffff800021a21ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff800021a21ff0) at sched_idle+0x27e
end trace frame: 0x0, count: -5
ddb{9}> mach ddbcpu 0xa
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800021a2aff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
pool_get(ffffffff8221e568,2) at pool_get+0x43
m_gethdr(2,1) at m_gethdr+0x3f
rtm_msg1(e,ffff800026e3cf70) at rtm_msg1+0x4c
rtm_ifchg(ffff8000005b3800) at rtm_ifchg+0x61
if_down(ffff8000005b3800) at if_down+0xa0
if_downall() at if_downall+0x5b
boot(104) at boot+0x99
reboot(104) at reboot+0x5b
panic(ffffffff81df855b) at panic+0x132
pool_do_get(ffffffff8221ebc8,2,ffff800026e3d294) at pool_do_get+0x309
pool_get(ffffffff8221ebc8,2) at pool_get+0x95
end trace frame: 0xffff800026e3d340, count: 0
ddb{10}> mach ddbcpu 0xb
Stopped at db_enter+0x10: popq %rbp
db_enter() at db_enter+0x10
panic(ffffffff81df855b) at panic+0x12a
pool_do_get(ffffffff8221e568,2,ffff800026e43294) at pool_do_get+0x309
pool_get(ffffffff8221e568,2) at pool_get+0x95
m_clget(0,2,802) at m_clget+0xdd
ixgbe_get_buf(ffff80000015c0e8,e) at ixgbe_get_buf+0xa3
ixgbe_rxfill(ffff80000015c0e8) at ixgbe_rxfill+0xae
ixgbe_queue_intr(ffff80000011ac40) at ixgbe_queue_intr+0x4f
intr_handler(ffff800026e434b0,ffff8000000cd700) at intr_handler+0x6e
Xintr_ioapic_edge4_untramp() at Xintr_ioapic_edge4_untramp+0x18f
acpicpu_idle() at acpicpu_idle+0x1ea
sched_idle(ffff800021a33ff0) at sched_idle+0x27e
end trace frame: 0x0, count: 3
