On Mon, Dec 22, 2025 at 1:15 PM Mark Kettenis <[email protected]> wrote:
[...]
> > Similar fault:
> >
> > kernel: protection fault trap, code=0
> > Stopped at Xipi_invlrange_pcid+0x37:
> >
> > ddb{0}> show reg
> > rdi 0xffffffff82ad6880 kernel_lock
> > rsi 0x1
> > rbp 0xffff800055ee1b70
> > rbx 0xffff800001d28100
> > rdx 0xffff800041d7f000
> > rcx 0
> > rax 0x800000000000
> > r8 0
> > r9 0
> > r10 0
> > r11 0x81df3a8602ca569e
> > r12 0xffff800055ee1bd0
> > r13 0xffff800001d28100
> > r14 0xffffffff82a11ff0 cpu_info_full_primary+0x1ff0
> > r15 0
> > rip 0xffffffff82578717 Xipi_invlrange_pcid+0x37
> > cs 0x8
> > rflags 0x10007 __ALIGN_SIZE+0xf007
> > rsp 0xffff800055ee1b10
> > ss 0
> > Xipi_invlrange_pcid+0x37:
>
> So we're in the loop, executing the first INVPCID instruction. At
> that point %rax is the address of the page we're currently
> invalidating and %rdx is the end of the range we're invalidating. The
> latter looks like a valid kernel address. But %rax is 0x800000000000.
> That is not a valid address. In fact it is not "canonical". And
> according to the documentation, that will generate a protection fault
> (#GP(0)).
>
> What I suspect is happening here is that somehow we end up with a
> userland address in %rax (read from tlb_shoot_addr1) and a kernel
> address in %rdx (read from tlb_shoot_addr2). We'll happily execute
> the loop, invalidating each userland page starting from
> tlb_shoot_addr1. Until we hit the end of userland address space at
> non-canonical address 0x800000000000. This may take a while if the
> start address is very low. Which explains why sometimes it appears
> that the machine just hangs.
>
> So the question is how the values we read from tlb_shoot_addr1 and
> tlb_shoot_addr2 become inconsistent. Note that the backtraces show
> that some CPUs are in pmap_do_remove(), shooting down userland
> addresses, and one CPU is in pmap_kremove() shooting down a kernel
> address. And there are CPUs in pmap_enter() as well, potentially
> shooting down a single userland page. But I don't immediately see how
> this can happen.
Another one, this time building lang/rust. The machine hung for 8+
hours, no protection fault trap this time. Forced a NMI to get into
ddb.
Thanks,
--Kor
NMI ... going to debugger
Stopped at __mp_lock+0x72: jmp __mp_lock+0x60
ddb{0}> show reg
rdi 0xffffffff82bb8f40 kernel_lock
rsi 0x1
rbp 0xffff80005635f660
rbx 0xffff800000a46480
rdx 0x848856 acpi_pdirpa+0x8346c7
rcx 0x848857 acpi_pdirpa+0x8346c8
rax 0xffffffff82a6fff0 cpu_info_full_primary+0x1ff0
r8 0x2
r9 0xce57d16c
r10 0xffff80005635f770
r11 0x9e9bb2e84196b4ee
r12 0xffff80005635f6c0
r13 0xffff800000a46480
r14 0xffffffff82a6fff0 cpu_info_full_primary+0x1ff0
r15 0
rip 0xffffffff82357512 __mp_lock+0x72
cs 0x8
rflags 0x297
rsp 0xffff80005635f650
ss 0
__mp_lock+0x72: jmp __mp_lock+0x60
ddb{0}> tr
__mp_lock(ffffffff82bb8f40) at __mp_lock+0x72
intr_handler(ffff80005635f6c0,ffff800000a46480) at intr_handler+0x71
Xintr_ioapic_edge16_untramp() at Xintr_ioapic_edge16_untramp+0x18f
Xspllower() at Xspllower+0x1d
ifiq_input(ffff80000192e460,ffff80005635f858) at ifiq_input+0x277
bge_rxeof(ffff80000192e000) at bge_rxeof+0x381
bge_intr(ffff80000192e000) at bge_intr+0x213
intr_handler(ffff80005635f970,ffff800001949b00) at intr_handler+0xa4
Xintr_ioapic_edge23_untramp() at Xintr_ioapic_edge23_untramp+0x18f
Xspllower() at Xspllower+0x1d
softintr_dispatch(0) at softintr_dispatch+0xd6
dosoftint(0) at dosoftint+0x3f
Xsoftclock() at Xsoftclock+0x27
alltraps() at alltraps+0x135
end of kernel
end trace frame: 0x7b7510950b0, count: -14
ddb{0}> ps /o
TID PID UID PRFLAGS PFLAGS CPU COMMAND
374116 23005 0 0x3 0 8 rustc
421350 7387 0 0x3 0x4000000 11 rustc
98132 3626 0 0x3 0x4000000 6 rustc
295744 71571 0 0x3 0x4000000 3 rustc
244684 51513 0 0x3 0x4000000 4 rustc
354800 48656 0 0x3 0x4000000 15 rustc
135269 32204 0 0x3 0x4000000 1 rustc
74683 6069 0 0x3 0x4000000 5 rustc
354152 53581 0 0x3 0x4000000 7 rustc
490408 51481 0 0x3 0x4000000 14 rustc
*113044 11201 0 0x3 0x4000000 0 rustc
202424 93051 0 0x3 0x4000000 12 rustc
54424 62600 0 0x3 0x4000000 13 rustc
468764 95748 0 0x3 0x4000000 10 rustc
486003 14751 0 0x3 0x4000000 9 cargo
ddb{0}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
23005 374116 28961 0 7 0x3 rustc
28961 129753 14751 0 3 0x83 wait rustc
7387 290694 1756 0 3 0x83 fsleep rustc
7387 143131 1756 0 3 0x4000083 piperd rustc
7387 421350 1756 0 7 0x4000003 rustc
7387 94043 1756 0 3 0x4000083 fsleep rustc
1756 522827 14751 0 3 0x83 wait rustc
3626 165363 79410 0 3 0x83 fsleep rustc
3626 282944 79410 0 3 0x4000083 piperd rustc
3626 98132 79410 0 7 0x4000003 rustc
3626 14905 79410 0 3 0x4000083 fsleep rustc
79410 91154 14751 0 3 0x83 wait rustc
71571 469479 20088 0 3 0x83 fsleep rustc
71571 379014 20088 0 3 0x4000083 piperd rustc
71571 295744 20088 0 7 0x4000003 rustc
71571 379664 20088 0 3 0x4000083 fsleep rustc
20088 418151 14751 0 3 0x83 wait rustc
51513 283797 13765 0 3 0x83 fsleep rustc
51513 442198 13765 0 3 0x4000083 piperd rustc
51513 294723 13765 0 3 0x4000083 fsleep rustc
51513 345895 13765 0 3 0x4000083 fsleep rustc
51513 496496 13765 0 3 0x4000083 fsleep rustc
51513 266070 13765 0 3 0x4000083 piperd rustc
51513 244684 13765 0 7 0x4000003 rustc
13765 387516 14751 0 3 0x83 wait rustc
48656 449662 74695 0 3 0x83 fsleep rustc
48656 434874 74695 0 3 0x4000083 piperd rustc
48656 354800 74695 0 7 0x4000003 rustc
48656 312726 74695 0 3 0x4000083 fsleep rustc
74695 46689 14751 0 3 0x83 wait rustc
32204 71010 3835 0 3 0x83 fsleep rustc
32204 330116 3835 0 3 0x4000083 piperd rustc
32204 135269 3835 0 7 0x4000003 rustc
32204 72019 3835 0 3 0x4000083 fsleep rustc
3835 307320 14751 0 3 0x83 wait rustc
6069 164458 75108 0 3 0x83 fsleep rustc
6069 505706 75108 0 3 0x4000083 piperd rustc
6069 74683 75108 0 7 0x4000003 rustc
6069 299467 75108 0 3 0x4000083 fsleep rustc
6069 485194 75108 0 3 0x4000083 piperd rustc
6069 250330 75108 0 3 0x4000083 fsleep rustc
75108 502927 14751 0 3 0x83 wait rustc
53581 265180 56977 0 3 0x83 fsleep rustc
53581 215893 56977 0 3 0x4000083 piperd rustc
53581 354152 56977 0 7 0x4000003 rustc
53581 475972 56977 0 3 0x4000083 fsleep rustc
56977 376438 14751 0 3 0x83 wait rustc
51481 117007 22981 0 3 0x83 fsleep rustc
51481 60956 22981 0 3 0x4000083 piperd rustc
51481 490408 22981 0 7 0x4000003 rustc
51481 133802 22981 0 3 0x4000083 fsleep rustc
22981 85096 14751 0 3 0x83 wait rustc
11201 353709 33442 0 3 0x83 fsleep rustc
11201 406120 33442 0 3 0x4000083 piperd rustc
*11201 113044 33442 0 7 0x4000003 rustc
11201 480980 33442 0 3 0x4000083 fsleep rustc
33442 19822 14751 0 3 0x83 wait rustc
93051 457381 17905 0 3 0x83 fsleep rustc
93051 297338 17905 0 3 0x4000083 piperd rustc
93051 374927 17905 0 3 0x4000083 fsleep rustc
93051 183538 17905 0 3 0x4000083 fsleep rustc
93051 140598 17905 0 3 0x4000083 fsleep rustc
93051 354551 17905 0 3 0x4000083 piperd rustc
93051 202424 17905 0 7 0x4000003 rustc
17905 246036 14751 0 3 0x83 wait rustc
62600 325886 49776 0 3 0x83 fsleep rustc
62600 488684 49776 0 3 0x4000083 piperd rustc
62600 435906 49776 0 3 0x4000083 fsleep rustc
62600 207725 49776 0 3 0x4000083 fsleep rustc
62600 29025 49776 0 3 0x4000083 fsleep rustc
62600 199013 49776 0 3 0x4000083 piperd rustc
62600 54424 49776 0 7 0x4000003 rustc
49776 427929 14751 0 3 0x83 wait rustc
95748 463687 46857 0 3 0x83 fsleep rustc
95748 422074 46857 0 3 0x4000083 piperd rustc
95748 31245 46857 0 3 0x4000083 fsleep rustc
95748 231189 46857 0 3 0x4000083 fsleep rustc
95748 68569 46857 0 3 0x4000083 piperd rustc
95748 136183 46857 0 3 0x4000083 fsleep rustc
95748 468764 46857 0 7 0x4000003 rustc
46857 105668 14751 0 3 0x83 wait rustc
14751 143761 12547 0 3 0x83 fsleep cargo
14751 333010 12547 0 3 0x4000083 piperd cargo
14751 329177 12547 0 3 0x4000083 kqread cargo
14751 130408 12547 0 3 0x4000083 kqread cargo
14751 308447 12547 0 3 0x4000083 kqread cargo
14751 362988 12547 0 3 0x4000083 kqread cargo
14751 447360 12547 0 3 0x4000083 kqread cargo
14751 201618 12547 0 3 0x4000083 kqread cargo
14751 322427 12547 0 3 0x4000083 kqread cargo
14751 51234 12547 0 3 0x4000083 kqread cargo
14751 489037 12547 0 3 0x4000083 kqread cargo
14751 465044 12547 0 3 0x4000083 kqread cargo
14751 440333 12547 0 3 0x4000083 kqread cargo
14751 486003 12547 0 7 0x4000003 cargo
14751 78955 12547 0 3 0x4000083 kqread cargo
14751 492960 12547 0 3 0x4000003 pipelk cargo
14751 213220 12547 0 3 0x4000083 kqread cargo
14751 15496 12547 0 3 0x4000083 kqread cargo
59199 125517 53254 0 3 0x100083 ttyout tail
12547 281797 64433 0 3 0x83 piperd bootstrap
64433 56055 15570 0 3 0x83 wait python3.13
15570 54944 20709 0 3 0x10008b sigsusp make
20709 416716 92009 0 3 0x10008b sigsusp make
92009 19107 9059 0 3 0x10008b sigsusp sh
9059 223492 60406 0 3 0x10008b sigsusp make
60406 143055 1 0 3 0x10008b sigsusp sh
87577 340426 1 0 3 0x100083 ttyin getty
81211 512956 1 0 3 0x100083 ttyin getty
65696 296159 1 0 3 0x100083 ttyin getty
88984 131360 1 0 3 0x100083 ttyin getty
48228 85947 1 0 3 0x100083 ttyin getty
53254 281796 1 0 3 0x10008b sigsusp ksh
94655 281992 1 0 3 0x100098 kqread cron
1030 114939 34907 95 3 0x1100092 kqread smtpd
35865 465676 34907 103 3 0x1100092 kqread smtpd
53220 195779 34907 95 3 0x1100092 kqread smtpd
62887 85568 34907 95 3 0x100092 kqread smtpd
52544 367703 34907 95 3 0x1100092 kqread smtpd
53913 186550 34907 95 3 0x1100092 kqread smtpd
34907 275126 1 0 3 0x100080 kqread smtpd
38436 281650 1 0 3 0x88 kqread sshd
5383 468041 1 0 3 0x100080 kqread ntpd
41721 196808 47686 83 3 0x100092 kqread ntpd
47686 444292 1 83 3 0x1100092 kqread ntpd
45659 19013 64764 74 3 0x1100092 bpf pflogd
64764 16593 1 0 3 0x80 sbwait pflogd
21188 15906 22956 73 3 0x1100090 kqread syslogd
22956 366479 1 0 3 0x100082 sbwait syslogd
78133 215670 0 0 3 0x14200 bored smr
16933 180221 0 0 3 0x14200 pgzero zerothread
87033 124305 0 0 3 0x14200 aiodoned aiodoned
28601 77736 0 0 3 0x14200 syncer update
47039 309237 0 0 3 0x14200 cleaner cleaner
80848 480359 0 0 3 0x14200 reaper reaper
72719 296135 0 0 3 0x14200 pgdaemon pagedaemon
4162 228258 0 0 3 0x14200 bored wsdisplay0
8804 452615 0 0 3 0x14200 usbtsk usbtask
58262 178765 0 0 3 0x14200 usbatsk usbatsk
90112 150350 0 0 3 0x40014200 acpi0 acpi0
95328 357213 0 0 7 0x40014200 idle31
47752 515087 0 0 7 0x40014200 idle30
86336 135479 0 0 7 0x40014200 idle29
11271 323170 0 0 7 0x40014200 idle28
48805 283253 0 0 7 0x40014200 idle27
37031 328739 0 0 7 0x40014200 idle26
29408 105774 0 0 7 0x40014200 idle25
87227 379131 0 0 7 0x40014200 idle24
44152 415398 0 0 7 0x40014200 idle23
75424 402591 0 0 7 0x40014200 idle22
55889 100292 0 0 7 0x40014200 idle21
86034 110942 0 0 7 0x40014200 idle20
5205 160573 0 0 7 0x40014200 idle19
37524 21293 0 0 7 0x40014200 idle18
87246 294655 0 0 7 0x40014200 idle17
48487 119090 0 0 7 0x40014200 idle16
9534 397493 0 0 3 0x40014200 idle15
88351 64295 0 0 3 0x40014200 idle14
1295 138468 0 0 3 0x40014200 idle13
20379 438265 0 0 3 0x40014200 idle12
9957 86261 0 0 3 0x40014200 idle11
68666 425386 0 0 3 0x40014200 idle10
99697 81208 0 0 3 0x40014200 idle9
36773 48512 0 0 3 0x40014200 idle8
80937 315201 0 0 3 0x40014200 idle7
90706 47138 0 0 3 0x40014200 idle6
77181 439798 0 0 3 0x40014200 idle5
52407 37635 0 0 3 0x40014200 idle4
50422 259492 0 0 3 0x40014200 idle3
54364 42251 0 0 3 0x40014200 idle2
2722 405728 0 0 3 0x40014200 idle1
12086 507396 0 0 3 0x14200 bored sensors
67156 35874 0 0 3 0x14200 bored softnet7
88576 127229 0 0 3 0x14200 bored softnet6
91100 87985 0 0 3 0x14200 bored softnet5
97040 88607 0 0 3 0x14200 bored softnet4
35457 287701 0 0 3 0x14200 bored softnet3
91090 189144 0 0 3 0x14200 bored softnet2
93950 104648 0 0 3 0x14200 bored softnet1
51196 166417 0 0 3 0x14200 bored softnet0
76547 192046 0 0 3 0x14200 bored systqmp
22835 378303 0 0 3 0x14200 bored systq
85789 268950 0 0 3 0x14200 tmoslp softclockmp
96314 239582 0 0 3 0x40014200 tmoslp softclock
21640 435319 0 0 3 0x40014200 idle0
1 120847 0 0 3 0x82 wait init
0 0 -1 0 3 0x10200 scheduler swapper
ddb{0}> show uvm
Current UVM status:
pagesize=4096 (0x1000), pagemask=0xfff, pageshift=12
32434530 VM pages: 434762 active, 940393 inactive, 1 wired, 23679372
free (3904138 zero)
freemin=1081151, free-target=1441534, inactive-target=0, wired-max=10811510
faults=77506444, traps=80971092, intrs=1397690, ctxswitch=134351661
fpuswitch=0
softint=4085523, syscalls=792974311, kmapent=10
fault counts:
noram=0, noanon=0, noamap=0, pgwait=0, pgrele=0
relocks=1742017(25765), upgrades=2553627(42818)
anget(retries)=10299753(0), amapcopy=5944400
neighbor anon/obj pg=465534/26365573, gets(lock/unlock)=11069143/1767805
cases: anon=10165663, anoncow=134090, obj=9189216, prcopy=1812700,
przero=56203762
daemon and swap counts:
woke=0, revs=0, scans=0, obscans=0, anscans=0
busy=0, freed=0, reactivate=0, deactivate=0
pageouts=0, pending=0, nswget=0
nswapdev=1
swpages=16777883, swpginuse=0, swpgonly=0 paging=0
kernel pointers:
objs(kern)=0xffffffff82ae2860
ddb{0}> mach cpuinfo
* 0: ddb
1: stopped
2: stopped
3: stopped
4: stopped
5: stopped
6: stopped
7: stopped
8: stopped
9: stopped
10: stopped
11: stopped
12: stopped
13: stopped
14: stopped
15: stopped
16: stopped
17: stopped
18: stopped
19: stopped
20: stopped
21: stopped
22: stopped
23: stopped
24: stopped
25: stopped
26: stopped
27: stopped
28: stopped
29: stopped
30: stopped
31: stopped
ddb{0}> mach ddb 0t1
Stopped at x86_ipi_db+0x16: leave
ddb{1}> show reg
rdi 0xffff8000552baff0
rsi 0
rbp 0xffff800056357300
rbx 0xffffffff82a32f68 ipifunc+0x38
rdx 0
rcx 0x7
rax 0xffffff7f
r8 0
r9 0
r10 0
r11 0x970841ab91a42885
r12 0x7
r13 0
r14 0xffff8000552baff0
r15 0
rip 0xffffffff8215ec56 x86_ipi_db+0x16
cs 0x8
rflags 0x282
rsp 0xffff8000563572f0
ss 0
x86_ipi_db+0x16: leave
ddb{1}> tr
x86_ipi_db(ffff8000552baff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffd948da5e048,369581e4000,14ff424000,3,22) at pmap_enter+0x662
uvm_fault_lower(ffff800056357668,ffff8000563576a0,ffff8000563575e0) at
uvm_fault_lower+0x255
uvm_fault(fffffd96b5e2be60,369581e4000,0,2) at uvm_fault+0x1c5
upageflttrap(ffff8000563577e0,369581e4000) at upageflttrap+0x6c
usertrap(ffff8000563577e0) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0x369d2c5e3e0, count: -9
ddb{1}> mach ddb 0t3
Stopped at x86_ipi_db+0x16: leave
ddb{3}> tr
x86_ipi_db(ffff8000552ccff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffd9c9feb7368,1709bd76000,155ca61000,3,22) at pmap_enter+0x662
uvm_fault_lower(ffff8000563bdc58,ffff8000563bdc90,ffff8000563bdbd0) at
uvm_fault_lower+0x255
uvm_fault(fffffd96b5e2bcf0,1709bd76000,0,2) at uvm_fault+0x1c5
upageflttrap(ffff8000563bddd0,1709bd76ff0) at upageflttrap+0x6c
usertrap(ffff8000563bddd0) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0x171298fda80, count: -9
ddb{3}> mach ddb 0t4
Stopped at x86_ipi_db+0x16: leave
ddb{4}> tr
x86_ipi_db(ffff8000552d5ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffda07ffdf6c0,9475df67000,1466707000,3,22) at pmap_enter+0x660
uvm_fault_lower(ffff8000561576b8,ffff8000561576f0,ffff800056157630) at
uvm_fault_lower+0x255
uvm_fault(fffffd94b089eb90,9475df67000,0,2) at uvm_fault+0x1c5
upageflttrap(ffff800056157830,9475df67350) at upageflttrap+0x6c
usertrap(ffff800056157830) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0x947f993c610, count: -9
ddb{4}> mach ddb 0t5
Stopped at x86_ipi_db+0x16: leave
ddb{5}> tr
x86_ipi_db(ffff8000552deff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffd948da5e120,9d9aff2b000,14dcb31000,3,22) at pmap_enter+0x662
uvm_fault_lower(ffff8000562a3138,ffff8000562a3170,ffff8000562a30b0) at
uvm_fault_lower+0x255
uvm_fault(fffffd9d890345e0,9d9aff2b000,0,2) at uvm_fault+0x1c5
upageflttrap(ffff8000562a32b0,9d9aff2b000) at upageflttrap+0x6c
usertrap(ffff8000562a32b0) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0x9d955f116c0, count: -9
ddb{5}> mach ddb 0t6
Stopped at x86_ipi_db+0x16: leave
ddb{6}> tr
x86_ipi_db(ffff8000552e7ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffda07ffdfaf8,21a9b239000,1d5267f000,3,21) at pmap_enter+0x662
uvm_fault_lower(ffff8000563c30e8,ffff8000563c3120,ffff8000563c3060) at
uvm_fault_lower+0x255
uvm_fault(fffffd944d822a28,21a9b239000,0,1) at uvm_fault+0x1c5
upageflttrap(ffff8000563c3260,21a9b239498) at upageflttrap+0x6c
usertrap(ffff8000563c3260) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0x21b2fe931e0, count: -9
ddb{6}> mach ddb 0t7
Stopped at x86_ipi_db+0x16: leave
ddb{7}> tr
x86_ipi_db(ffff8000552f0ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffd948da5ec18,82720b48000,1558a4f000,3,22) at pmap_enter+0x662
uvm_fault_lower(ffff8000560b1c38,ffff8000560b1c70,ffff8000560b1bb0) at
uvm_fault_lower+0x255
uvm_fault(fffffd944d8228b8,82720b48000,0,2) at uvm_fault+0x1c5
upageflttrap(ffff8000560b1db0,82720b48000) at upageflttrap+0x6c
usertrap(ffff8000560b1db0) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0x827c9dd0480, count: -9
ddb{7}> mach ddb 0t8
Stopped at x86_ipi_db+0x16: leave
ddb{8}> tr
x86_ipi_db(ffff8000552f9ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffd948da5ea68,41b33562000,1682966000,4,24) at pmap_enter+0x662
uvm_fault_lower(ffff800056171718,ffff800056171750,ffff800056171690) at
uvm_fault_lower+0x255
uvm_fault(fffffd9ca038b458,41b33562000,0,4) at uvm_fault+0x1c5
upageflttrap(ffff800056171890,41b33562460) at upageflttrap+0x6c
usertrap(ffff800056171890) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0x72e892111be0, count: -9
ddb{8}> mach ddb 0t9
Stopped at x86_ipi_db+0x16: leave
ddb{9}> tr
x86_ipi_db(ffff800055302ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffd9d8f7d2448,3dd9775b000,14cb0de000,3,22) at pmap_enter+0x5e2
uvm_fault_upper(ffff800056465738,ffff800056465770,ffff800056465630) at
uvm_fault_upper+0x1e0
uvm_fault(fffffd9ca038ba18,3dd9775b000,0,2) at uvm_fault+0xce
upageflttrap(ffff8000564658b0,3dd9775b0d0) at upageflttrap+0x6c
usertrap(ffff8000564658b0) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0x3dda5b5cc10, count: -9
ddb{9}> mach ddb 0t10
Stopped at x86_ipi_db+0x16: leave
ddb{10}> tr
x86_ipi_db(ffff80005530bff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffd9c9feb7a28,3ff11c91000,14e72cf000,3,22) at pmap_enter+0x662
uvm_fault_lower(ffff8000562353b8,ffff8000562353f0,ffff800056235330) at
uvm_fault_lower+0x255
uvm_fault(fffffd96b5e2b8a0,3ff11c91000,0,2) at uvm_fault+0x1c5
upageflttrap(ffff800056235530,3ff11c91010) at upageflttrap+0x6c
usertrap(ffff800056235530) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0x3ff355cdcb0, count: -9
ddb{10}> mach ddb 0t11
Stopped at x86_ipi_db+0x16: leave
ddb{11}> tr
x86_ipi_db(ffff800055314ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffd948da5eea0,2a41b438000,148348d000,3,22) at pmap_enter+0x66a
uvm_fault_lower(ffff800056279188,ffff8000562791c0,ffff800056279100) at
uvm_fault_lower+0x255
uvm_fault(fffffd94b089e460,2a41b438000,0,2) at uvm_fault+0x1c5
upageflttrap(ffff800056279300,2a41b438600) at upageflttrap+0x6c
usertrap(ffff800056279300) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0x2a443d28fa0, count: -9
ddb{11}> mach ddb 0t12
Stopped at x86_ipi_db+0x16: leave
ddb{12}> tr
x86_ipi_db(ffff80005531dff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_do_remove(fffffd948da5e480,3003d1f5000,3003d1f6000,0) at
pmap_do_remove+0x592
uvm_unmap_kill_entry_withlock(fffffd944d822468,fffffd94d09ae060,1) at
uvm_unmap_kill_entry_withlock+0x133
uvm_unmap_remove(fffffd944d822468,3003d1f5000,3003d1f6000,ffff8000561d3600,0,1,a803e8cf8edef55b)
at uvm_unmap_remove+0x32f
sys_munmap(ffff800055ffb510,ffff8000561d3700,ffff8000561d3680) at
sys_munmap+0x10b
syscall(ffff8000561d3700) at syscall+0x5f9
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x300b476f550, count: -9
ddb{12}> mach ddb 0t13
Stopped at x86_ipi_db+0x16: leave
ddb{13}> tr
x86_ipi_db(ffff800055326ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffda07ffdf438,bc15e2e2000,143c866000,3,22) at pmap_enter+0x66a
uvm_fault_lower(ffff80005646bca8,ffff80005646bce0,ffff80005646bc20) at
uvm_fault_lower+0x255
uvm_fault(fffffd94b089e740,bc15e2e2000,0,2) at uvm_fault+0x1c5
upageflttrap(ffff80005646be20,bc15e2e2000) at upageflttrap+0x6c
usertrap(ffff80005646be20) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0xbc0f1c4fc40, count: -9
ddb{13}> mach ddb 0t14
Stopped at x86_ipi_db+0x16: leave
ddb{14}> tr
x86_ipi_db(ffff80005532fff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffd948da5e708,7feed201000,15116c8000,3,22) at pmap_enter+0x66a
uvm_fault_lower(ffff800056477cb8,ffff800056477cf0,ffff800056477c30) at
uvm_fault_lower+0x255
uvm_fault(fffffd944d822b98,7feed201000,0,2) at uvm_fault+0x1c5
upageflttrap(ffff800056477e30,7feed201ff0) at upageflttrap+0x6c
usertrap(ffff800056477e30) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0x7fee39d2ba0, count: -9
ddb{14}> mach ddb 0t15
Stopped at x86_ipi_db+0x16: leave
ddb{15}> tr
x86_ipi_db(ffff800055338ff0) at x86_ipi_db+0x16
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
pmap_enter(fffffda07ffdf798,ab7a005a000,14b7096000,3,22) at pmap_enter+0x662
uvm_fault_lower(ffff8000562f1978,ffff8000562f19b0,ffff8000562f18f0) at
uvm_fault_lower+0x255
uvm_fault(fffffd96b5e2b000,ab7a005a000,0,2) at uvm_fault+0x1c5
upageflttrap(ffff8000562f1af0,ab7a005a000) at upageflttrap+0x6c
usertrap(ffff8000562f1af0) at usertrap+0x28b
recall_trap() at recall_trap+0x8
end of kernel
end trace frame: 0xab816aad490, count: -9
ddb{15}>
[...]
