On 22/12/25(Mon) 14:59, Martin Pieuchot wrote:
> "K R" can reproduce a hang on a multiple sockets amd64 that has been
> first reported in a different thread:
> https://marc.info/?l=openbsd-tech&m=176631121132731&w=2
>
> The reports seem to always contain a CPU spinning for `tlb_shoot_wait'
> inside pmap_kremove().
>
> In two these reports, including the one below, instead of a hang, a CPU
> faulted inside Xipi_invlrange_pcid executing the following instruction:
>
> 00000000000006e0 <Xipi_invlrange_pcid>:
> [...]
> 717: 66 0f 38 82 0c 24 invpcid (%rsp),%rcx
>
> Any idea?
Here's another report:
The machine hung as usual -- I waited a couple of days but no
protection fault this time. Forced a NMI to get a ddb prompt.
Thanks,
--Kor
NMI ... going to debugger
Stopped at __mp_lock+0x72: jmp __mp_lock+0x60
ddb{0}> show reg
rdi 0xffffffff82a73fc0 kernel_lock
rsi 0x2
rbp 0xffff800055bbd400
rbx 0xffff800000a46480
rdx 0x24e6b4 acpi_pdirpa+0x23a525
rcx 0x24e6b5 acpi_pdirpa+0x23a526
rax 0xffffffff82a1bff0 cpu_info_full_primary+0x1ff0
r8 0
r9 0
r10 0
r11 0x2a63abab505df87
r12 0xffff800055bbd460
r13 0xffff800000a46480
r14 0xffffffff82a1bff0 cpu_info_full_primary+0x1ff0
r15 0
rip 0xffffffff810a9f92 __mp_lock+0x72
cs 0x8
rflags 0x297
rsp 0xffff800055bbd3f0
ss 0
__mp_lock+0x72: jmp __mp_lock+0x60
ddb{0}> tr
__mp_lock(ffffffff82a73fc0) at __mp_lock+0x72
intr_handler(ffff800055bbd460,ffff800000a46480) at intr_handler+0x71
Xintr_ioapic_edge16_untramp() at
Xintr_ioapic_edge16_untramp+0x18f_kernel_lock() at _kernel_lock+0xb2
softintr_dispatch(0) at softintr_dispatch+0xeb
dosoftint(0) at dosoftint+0x3f
Xsoftclock() at Xsoftclock+0x27
acpicpu_idle() at acpicpu_idle+0x228
sched_idle(ffffffff82a1bff0) at sched_idle+0x24b
end trace frame: 0x0, count: -9
ddb{0}> ps /o
TID PID UID PRFLAGS PFLAGS CPU COMMAND
305318 98615 0 0x3 0 15 nfdump
135921 98615 0 0x3 0x4000000 4 nfdump
487013 72983 0 0x3 0x4000000 1 nfdump
518264 72983 0 0x3 0x4000000 7 nfdump
22530 98238 0 0x3 0x4000000 6 nfdump
96696 98238 0 0x3 0x4000000 3 nfdump
285208 60544 0 0x14000 0x200 5 softnet0
490656 92097 0 0x14000 0x200 2K systqmp
ddb{0}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
98615 305318 29768 0 7 0x3 nfdump
98615 513109 29768 0 3 0x4000083 fsleep nfdump
98615 34706 29768 0 3 0x4000003 vmmaplk nfdump
98615 473283 29768 0 3 0x4000083 fsleep nfdump
98615 144335 29768 0 3 0x4000083 fsleep nfdump
98615 408475 29768 0 3 0x4000083 fsleep nfdump
98615 453055 29768 0 3 0x4000083 fsleep nfdump
98615 135921 29768 0 7 0x4000003 nfdump
72983 253626 65817 0 3 0x3 vmmaplk nfdump
72983 182758 65817 0 3 0x4000083 fsleep nfdump
72983 487013 65817 0 7 0x4000003 nfdump
72983 359 65817 0 3 0x4000083 fsleep nfdump
72983 65895 65817 0 3 0x4000083 fsleep nfdump
72983 225535 65817 0 3 0x4000083 fsleep nfdump
72983 137053 65817 0 3 0x4000083 fsleep nfdump
72983 518264 65817 0 7 0x4000003 nfdump
98238 363429 46986 0 3 0x83 fsleep nfdump
98238 264903 46986 0 3 0x4000083 fsleep nfdump
98238 22530 46986 0 7 0x4000003 nfdump
98238 295466 46986 0 3 0x4000083 fsleep nfdump
98238 5573 46986 0 3 0x4000083 fsleep nfdump
98238 109865 46986 0 3 0x4000083 fsleep nfdump
98238 305747 46986 0 3 0x4000083 fsleep nfdump
98238 96696 46986 0 7 0x4000003 nfdump
46986 462433 1 0 3 0x10008b sigsusp sh
29768 215589 1 0 3 0x10008b sigsusp sh
65817 67721 1 0 3 0x10008b sigsusp sh
22709 260287 1 0 3 0x100083 ttyin getty
68087 176928 1 0 3 0x100083 ttyin getty
93467 113919 1 0 3 0x100083 ttyin getty
68961 517132 1 0 3 0x100083 ttyin getty
37832 466117 1 0 3 0x100083 ttyin getty
93297 248414 1 0 3 0x100083 ttyin ksh
19083 458404 1 0 3 0x100098 kqread cron
42641 195044 1 99 3 0x1100090 kqread sndiod
94202 425435 1 110 3 0x100090 kqread sndiod
69510 270817 12416 95 3 0x1100092 kqread smtpd
12748 235385 12416 103 3 0x1100092 kqread smtpd
35924 38404 12416 95 3 0x1100092 kqread smtpd
78550 81862 12416 95 3 0x100092 kqread smtpd
25166 478139 12416 95 3 0x1100092 kqread smtpd
54294 92515 12416 95 3 0x1100092 kqread smtpd
12416 488676 1 0 3 0x100080 kqread smtpd
47538 434887 1 0 3 0x88 kqread sshd
79798 342483 1 0 3 0x100080 kqread ntpd
88803 336519 13546 83 3 0x100092 kqread ntpd
13546 452696 1 83 3 0x1100092 kqread ntpd
75205 502543 10572 74 3 0x1100092 bpf pflogd
10572 395093 1 0 3 0x80 sbwait pflogd
13858 200359 92046 73 3 0x1100090 kqread syslogd
92046 320237 1 0 3 0x100082 sbwait syslogd
51409 288733 1 0 3 0x100080 kqread resolvd
21597 336014 98313 77 3 0x100092 kqread dhcpleased
23003 330276 98313 77 3 0x100092 kqread dhcpleased
98313 217711 1 0 3 0x80 kqread dhcpleased
73031 278447 63327 115 3 0x100092 kqread slaacd
9758 283338 63327 115 3 0x100092 kqread slaacd
63327 435718 1 0 3 0x100080 kqread slaacd
95153 439972 0 0 3 0x14200 bored smr
52703 157354 0 0 3 0x14200 pgzero zerothread
13203 296610 0 0 3 0x14200 aiodoned aiodoned
21439 508479 0 0 3 0x14200 syncer update
80497 220144 0 0 3 0x14200 cleaner cleaner
85694 511787 0 0 3 0x14200 reaper reaper
67415 114240 0 0 3 0x14200 pgdaemon pagedaemon
77798 448165 0 0 3 0x14200 bored wsdisplay0
37050 259221 0 0 3 0x14200 usbtsk usbtask
69711 158396 0 0 3 0x14200 usbatsk usbatsk
77529 364020 0 0 3 0x40014200 acpi0 acpi0
71670 81324 0 0 7 0x40014200 idle31
54355 410074 0 0 7 0x40014200 idle30
50133 154915 0 0 7 0x40014200 idle29
69349 216845 0 0 7 0x40014200 idle28
35847 336739 0 0 7 0x40014200 idle27
84440 510422 0 0 7 0x40014200 idle26
53662 343465 0 0 7 0x40014200 idle25
62443 496534 0 0 7 0x40014200 idle24
54916 422487 0 0 7 0x40014200 idle23
72411 246197 0 0 7 0x40014200 idle22
62845 444995 0 0 7 0x40014200 idle21
93790 397160 0 0 7 0x40014200 idle20
49908 70875 0 0 7 0x40014200 idle19
59858 421658 0 0 7 0x40014200 idle18
55760 417386 0 0 7 0x40014200 idle17
97852 91881 0 0 7 0x40014200 idle16
89403 122784 0 0 3 0x40014200 idle15
41682 283230 0 0 7 0x40014200 idle14
90580 40109 0 0 7 0x40014200 idle13
66933 125104 0 0 7 0x40014200 idle12
90130 514210 0 0 7 0x40014200 idle11
24541 233817 0 0 7 0x40014200 idle10
89558 343110 0 0 7 0x40014200 idle9
19900 333321 0 0 7 0x40014200 idle8
87381 326731 0 0 3 0x40014200 idle7
4757 77875 0 0 3 0x40014200 idle6
27834 31596 0 0 3 0x40014200 idle5
36824 261782 0 0 3 0x40014200 idle4
67533 468294 0 0 3 0x40014200 idle3
19818 428844 0 0 3 0x40014200 idle2
11397 331255 0 0 3 0x40014200 idle1
64538 497188 0 0 2 0x40014200 sensors
58867 36401 0 0 3 0x14200 bored softnet7
67873 36860 0 0 3 0x14200 bored softnet6
91799 194444 0 0 3 0x14200 bored softnet5
61636 308573 0 0 3 0x14200 bored softnet4
82037 454164 0 0 3 0x14200 bored softnet3
3192 466309 0 0 3 0x14200 bored softnet2
45133 422696 0 0 3 0x14200 bored softnet1
60544 285208 0 0 7 0x14200 softnet0
92097 490656 0 0 7 0x14200 systqmp
94930 294160 0 0 3 0x14200 bored systq
9833 354990 0 0 3 0x14200 tmoslp softclockmp
1327 222666 0 0 3 0x40014200 tmoslp softclock
*32628 328970 0 0 7 0x40014200 idle0
1 31056 0 0 3 0x82 wait init
0 0 -1 0 3 0x10200 scheduler swapper
ddb{0}> mach cpuinfo
* 0: ddb
1: stopped
2: stopped
3: stopped
4: stopped
5: stopped
6: stopped
7: stopped
8: stopped
9: stopped
10: stopped
11: stopped
12: stopped
13: stopped
14: stopped
15: stopped
16: stopped
17: stopped
18: stopped
19: stopped
20: stopped
21: stopped
22: stopped
23: stopped
24: stopped
25: stopped
26: stopped
27: stopped
28: stopped
29: stopped
30: stopped
31: stopped
ddb{0}> mach ddb 0t1
Stopped at x86_ipi_db+0x16: leave
ddb{1}> tr
x86_ipi_db(ffff8000552baff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffd9c9b5b1e68,f5ccb699000,197a2b8000,3,22) at pmap_enter+0x662
uvm_fault_lower(ffff800055e6ba68,ffff800055e6baa0,ffff800055e6b9e0) at
uvm_fault_lower+0x255
uvm_fault(fffffd9da0e22460,f5ccb699000,0,2) at uvm_fault+0x1c5
upageflttrap(ffff800055e6bbe0,f5ccb699000) at upageflttrap+0x6c
usertrap(ffff800055e6bbe0) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0xf5d79a1b7b0, count: -9
ddb{1}> mach ddb 0t2
Stopped at x86_ipi_db+0x16: leave
ddb{2}> tr
x86_ipi_db(ffff8000552c3ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_tlb_shootrange(ffffffff82ba4d60,ffff800055d18000,ffff800055d1a000,1)
at pmap_tlb_shootrange+0x2a2
pmap_kremove(ffff800055d18000,2000) at pmap_kremove+0x81
km_free(ffff800055d18000,2000,ffff800055bd4fc0,ffffffff82912e28) at
km_free+0x1c6
pool_multi_free_ni(ffffffff82b61478,ffff800055d18000) at pool_multi_free_ni+0x82
pool_p_free(ffffffff82b61478,fffffda07f7e2a10) at pool_p_free+0x75
pool_gc_pages(0) at pool_gc_pages+0x1d0
taskq_thread(ffffffff82a51248) at taskq_thread+0x129
end trace frame: 0x0, count: -10
ddb{2}> mach ddb 0t3
Stopped at x86_ipi_db+0x16: leave
ddb{3}> tr
x86_ipi_db(ffff8000552ccff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffd9c9b5b1cb8,40881c7c000,19773df000,3,22) at pmap_enter+0x662
uvm_fault_lower(ffff800055eece98,ffff800055eeced0,ffff800055eece10) at
uvm_fault_lower+0x255
uvm_fault(fffffd9da0e22e70,40881c7c000,0,2) at uvm_fault+0x1c5
upageflttrap(ffff800055eed010,40881c7c000) at upageflttrap+0x6c
usertrap(ffff800055eed010) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0x408fb4d17b0, count: -9
ddb{3}> mach ddb 0t4
Stopped at x86_ipi_db+0x16: leave
ddb{4}> tr
x86_ipi_db(ffff8000552d5ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_do_remove(fffffd9d56d6f440,1a9d2568000,1a9d2570000,0) at
pmap_do_remove+0x3d2
uvm_unmap_kill_entry_withlock(fffffd9c9a3dba28,fffffd9c61094998,1) at
uvm_unmap_kill_entry_withlock+0x133
uvm_unmap_remove(fffffd9c9a3dba28,1a9d2568000,1a9d2a68000,ffff800055ef9b60,0,1,1b9fdbf60bd62a88)
at uvm_unmap_remove+0x32f
sys_munmap(ffff800055e737a0,ffff800055ef9c60,ffff800055ef9be0) at
sys_munmap+0x10b
syscall(ffff800055ef9c60) at syscall+0x5f9
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x1a9145d8b90, count: -9
ddb{4}> mach ddb 0t5
Stopped at x86_ipi_db+0x16: leave
ddb{5}> tr
x86_ipi_db(ffff8000552deff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
_kernel_lock() at _kernel_lock+0xb2
rt_clone(ffff800055bdb418,ffff800055bdb4a8,0) at rt_clone+0x64
rtalloc(ffff800055bdb4a8,1,0) at rtalloc+0x69
in_arpinput(ffff80000192e048,fffffd800441f600) at in_arpinput+0x171
arpintr() at arpintr+0xb7
if_netisr(0) at if_netisr+0xd5
taskq_thread(ffff800000037000) at taskq_thread+0x129
end trace frame: 0x0, count: -10
ddb{5}> mach ddb 0t6
Stopped at x86_ipi_db+0x16: leave
ddb{6}> tr
x86_ipi_db(ffff8000552e7ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_page_remove(fffffd81bbe67b00) at pmap_page_remove+0x342
uvm_anfree(fffffd9c3c9de440) at uvm_anfree+0x61
amap_wipeout(fffffd9c3f45acc0) at amap_wipeout+0x196
uvm_unmap_detach(ffff800055e65880,0) at uvm_unmap_detach+0xbe
sys_munmap(ffff800055d7efc0,ffff800055e65980,ffff800055e65900) at
sys_munmap+0x185
syscall(ffff800055e65980) at syscall+0x5f9
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x408ec4521b0, count: -10
ddb{6}> mach ddb 0t7
Stopped at x86_ipi_db+0x16: leave
ddb{7}> tr
x86_ipi_db(ffff8000552f0ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffd9c9b5b1e68,f5d3388f000,197942e000,3,22) at pmap_enter+0x662
uvm_fault_lower(ffff800055ef34b8,ffff800055ef34f0,ffff800055ef3430) at
uvm_fault_lower+0x255
uvm_fault(fffffd9da0e22460,f5d3388f000,0,2) at uvm_fault+0x1c5
upageflttrap(ffff800055ef3630,f5d3388f000) at upageflttrap+0x6c
usertrap(ffff800055ef3630) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0xf5cab2049d0, count: -9
ddb{7}> mach ddb 0t15
Stopped at x86_ipi_db+0x16: leave
ddb{15}> tr
x86_ipi_db(ffff800055338ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
_kernel_lock() at _kernel_lock+0xb2
postsig(ffff800055d7d268,b,ffff800055e17210) at postsig+0x3d2
userret(ffff800055d7d268) at userret+0x15d
usertrap(ffff800055e172c0) at usertrap+0x2c8
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0x7d1c39d94d80, count: -8
ddb{15}>
[...]
