We are having regular panics on several machines in the cluster. Below follows the script from the kgdb(1) session, hopefully providing enough information. This machine runs 11.0-CURRENT #2 r262892, from 2 days ago.
It uses tmpfs(5) for the port build workspace. I have an unconfirmed
suspicion that use of sysutils/lsof is involved somehow, but cannot be
sure. (In my experience with panics with port building, removing lsof
from the system did have an effect, but I may be going down the wrong
rabbit hole.)
Script started on Sun Mar 9 16:40:07 2014
r...@redbuild01.nyi:/usr/obj/usr/src/sys/REDBUILD # sh
# kgdb ./kernel.debug /var/crash/vmcore.1
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "amd64-marcel-freebsd"...
Unread portion of the kernel message buffer:
panic: vm_fault: fault on nofault entry, addr: fffffe035021a000
cpuid = 1
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe1839a54180
kdb_backtrace() at kdb_backtrace+0x39/frame 0xfffffe1839a54230
panic() at panic+0x155/frame 0xfffffe1839a542b0
vm_fault_hold() at vm_fault_hold+0x1e7a/frame 0xfffffe1839a54500
vm_fault() at vm_fault+0x77/frame 0xfffffe1839a54540
trap_pfault() at trap_pfault+0x199/frame 0xfffffe1839a545e0
trap() at trap+0x4a0/frame 0xfffffe1839a547f0
calltrap() at calltrap+0x8/frame 0xfffffe1839a547f0
--- trap 0xc, rip = 0xffffffff80d97bab, rsp = 0xfffffe1839a548b0, rbp =
0xfffffe1839a54910 ---
copyout() at copyout+0x3b/frame 0xfffffe1839a54910
memrw() at memrw+0x19f/frame 0xfffffe1839a54950
giant_read() at giant_read+0xa4/frame 0xfffffe1839a54990
devfs_read_f() at devfs_read_f+0xeb/frame 0xfffffe1839a549f0
dofileread() at dofileread+0x95/frame 0xfffffe1839a54a40
kern_readv() at kern_readv+0x68/frame 0xfffffe1839a54a90
sys_read() at sys_read+0x63/frame 0xfffffe1839a54ae0
amd64_syscall() at amd64_syscall+0x3fb/frame 0xfffffe1839a54bf0
Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe1839a54bf0
--- syscall (3, FreeBSD ELF64, sys_read), rip = 0x800b8444a, rsp =
0x7fffffffd088, rbp = 0x7fffffffd0d0 ---
KDB: enter: panic
Reading symbols from /boot/kernel/zfs.ko.symbols...done.
Loaded symbols for /boot/kernel/zfs.ko.symbols
Reading symbols from /boot/kernel/opensolaris.ko.symbols...done.
Loaded symbols for /boot/kernel/opensolaris.ko.symbols
Reading symbols from /boot/kernel/ums.ko.symbols...done.
Loaded symbols for /boot/kernel/ums.ko.symbols
Reading symbols from /boot/kernel/tmpfs.ko.symbols...done.
Loaded symbols for /boot/kernel/tmpfs.ko.symbols
Reading symbols from /boot/kernel/nullfs.ko.symbols...done.
Loaded symbols for /boot/kernel/nullfs.ko.symbols
Reading symbols from /boot/kernel/linprocfs.ko.symbols...done.
Loaded symbols for /boot/kernel/linprocfs.ko.symbols
Reading symbols from /boot/kernel/linux.ko.symbols...done.
Loaded symbols for /boot/kernel/linux.ko.symbols
#0 doadump (textdump=-967130448) at pcpu.h:219
219 __asm("movq %%gs:%1,%0" : "=r" (td)
(kgdb) bt
#0 doadump (textdump=-967130448) at pcpu.h:219
#1 0xffffffff8034a1a5 in db_fncall (dummy1=<value optimized out>,
dummy2=<value optimized out>, dummy3=<value optimized out>, dummy4=<value
optimized out>)
at /usr/src/sys/ddb/db_command.c:578
#2 0xffffffff80349e8d in db_command (cmd_table=0x0) at
/usr/src/sys/ddb/db_command.c:449
#3 0xffffffff80349c04 in db_command_loop () at
/usr/src/sys/ddb/db_command.c:502
#4 0xffffffff8034c660 in db_trap (type=<value optimized out>, code=0)
at /usr/src/sys/ddb/db_main.c:231
#5 0xffffffff80987ae9 in kdb_trap (type=3, code=0, tf=<value optimized out>)
at /usr/src/sys/kern/subr_kdb.c:656
#6 0xffffffff80d999b9 in trap (frame=0xfffffe1839a54160)
at /usr/src/sys/amd64/amd64/trap.c:571
#7 0xffffffff80d7e6e2 in calltrap () at
/usr/src/sys/amd64/amd64/exception.S:231
#8 0xffffffff8098724e in kdb_enter (why=0xffffffff8100f4ba "panic", msg=<value
optimized out>)
at cpufunc.h:63
#9 0xffffffff80946a75 in panic (fmt=<value optimized out>)
at /usr/src/sys/kern/kern_shutdown.c:752
#10 0xffffffff80c0a1fa in vm_fault_hold (map=<value optimized out>,
vaddr=<value optimized out>, fault_type=<value optimized out>,
fault_flags=<value optimized out>, m_hold=<value optimized out>)
at /usr/src/sys/vm/vm_fault.c:272
#11 0xffffffff80c08337 in vm_fault (map=0xfffff80002000000, vaddr=<value
optimized out>,
fault_type=1 '\001', fault_flags=128) at /usr/src/sys/vm/vm_fault.c:217
#12 0xffffffff80d9a1a9 in trap_pfault (frame=0xfffffe1839a54800, usermode=0)
at /usr/src/sys/amd64/amd64/trap.c:767
#13 0xffffffff80d999d0 in trap (frame=0xfffffe1839a54800)
at /usr/src/sys/amd64/amd64/trap.c:455
#14 0xffffffff80d7e6e2 in calltrap () at
/usr/src/sys/amd64/amd64/exception.S:231
#15 0xffffffff80d97bab in copyout () at /usr/src/sys/amd64/amd64/support.S:246
#16 0xffffffff8099c2f5 in uiomove_faultflag (cp=<value optimized out>,
n=<value optimized out>, uio=0xfffffe1839a54ab0, nofault=<value optimized
out>)
at /usr/src/sys/kern/subr_uio.c:192
#17 0xffffffff80d8612f in memrw (dev=0xfffff8000dbd0400,
uio=0xfffffe1839a54ab0,
flags=113246208) at /usr/src/sys/amd64/amd64/mem.c:101
#18 0xffffffff808ecf04 in giant_read (dev=0xfffff8000dbd0400,
uio=0xfffffe1839a54ab0, ioflag=0)
at /usr/src/sys/kern/kern_conf.c:442
#19 0xffffffff808185cb in devfs_read_f (fp=0xfffff80083439230,
uio=0xfffffe1839a54ab0,
cred=<value optimized out>, flags=0, td=0xfffff80e4edb8490)
at /usr/src/sys/fs/devfs/devfs_vnops.c:1193
#20 0xffffffff809a15e5 in dofileread (td=0xfffff80e4edb8490, fd=4,
fp=0xfffff80083439230,
auio=0xfffffe1839a54ab0, offset=<value optimized out>, flags=1172307968) at
file.h:299
#21 0xffffffff809a1308 in kern_readv (td=0xfffff80e4edb8490, fd=4,
auio=0xfffffe1839a54ab0)
at /usr/src/sys/kern/sys_generic.c:256
#22 0xffffffff809a1293 in sys_read (td=<value optimized out>, uap=<value
optimized out>)
at /usr/src/sys/kern/sys_generic.c:171
#23 0xffffffff80d9a9fb in amd64_syscall (td=0xfffff80e4edb8490, traced=0) at
subr_syscall.c:133
#24 0xffffffff80d7e9cb in Xfast_syscall () at
/usr/src/sys/amd64/amd64/exception.S:390
#25 0x0000000800b8444a in ?? ()
Previous frame inner to this frame (corrupt stack?)
Current language: auto; currently minimal
(kgdb) frame 19
#19 0xffffffff808185cb in devfs_read_f (fp=0xfffff80083439230,
uio=0xfffffe1839a54ab0,
cred=<value optimized out>, flags=0, td=0xfffff80e4edb8490)
at /usr/src/sys/fs/devfs/devfs_vnops.c:1193
1193 error = dsw->d_read(dev, uio, ioflag);
(kgdb) list
1188 ioflag = fp->f_flag & (O_NONBLOCK | O_DIRECT);
1189 if (ioflag & O_DIRECT)
1190 ioflag |= IO_DIRECT;
1191
1192 foffset_lock_uio(fp, uio, flags | FOF_NOLOCK);
1193 error = dsw->d_read(dev, uio, ioflag);
1194 if (uio->uio_resid != resid || (error == 0 && resid != 0))
1195 vfs_timestamp(&dev->si_atime);
1196 td->td_fpop = fpop;
1197 dev_relthread(dev, ref);
(kgdb) down
#18 0xffffffff808ecf04 in giant_read (dev=0xfffff8000dbd0400,
uio=0xfffffe1839a54ab0, ioflag=0)
at /usr/src/sys/kern/kern_conf.c:442
442 retval = dsw->d_gianttrick->d_read(dev, uio, ioflag);
(kgdb) list
437
438 dsw = dev_refthread(dev, &ref);
439 if (dsw == NULL)
440 return (ENXIO);
441 mtx_lock(&Giant);
442 retval = dsw->d_gianttrick->d_read(dev, uio, ioflag);
443 mtx_unlock(&Giant);
444 dev_relthread(dev, ref);
445 return (retval);
446 }
(kgdb) p *dev
$1 = {si_spare0 = 0x0, si_flags = 4, si_atime = {tv_sec = 1394286776, tv_nsec =
0},
si_ctime = {tv_sec = 1394236183, tv_nsec = 584945000}, si_mtime = {tv_sec =
1394236183,
tv_nsec = 584945000}, si_uid = 0, si_gid = 2, si_mode = 416, si_cred = 0x0,
si_drv0 = 1,
si_refcount = 9, si_list = {le_next = 0xfffff8000dbd0600, le_prev =
0xffffffff8144db18},
si_clone = {le_next = 0x0, le_prev = 0x0}, si_children = {lh_first = 0x0},
si_siblings = {
le_next = 0x0, le_prev = 0x0}, si_parent = 0x0, si_mountpt = 0x0, si_drv1 =
0x0,
si_drv2 = 0x0, si_devsw = 0xffffffff8144da78, si_iosize_max = 65536,
si_usecount = 1,
si_threadcount = 2, __si_u = {__sid_snapdata = 0x0},
si_name = "kmem", '\0' <repeats 59 times>}
(kgdb) p *uio
$2 = {uio_iov = 0xfffffe1839a54aa0, uio_iovcnt = 1, uio_offset =
-2184830705664,
uio_resid = 113246208, uio_segflg = UIO_USERSPACE, uio_rw = UIO_READ,
uio_td = 0xfffff80e4edb8490}
(kgdb) p *ioflag
Cannot access memory at address 0x0
(kgdb) p Giant
$3 = {lock_object = {lo_name = 0xffffffff8100e05a "Giant", lo_flags = 17498112,
lo_data = 0,
lo_witness = 0x0}, mtx_lock = 18446735339069080720}
(kgdb) down
#17 0xffffffff80d8612f in memrw (dev=0xfffff8000dbd0400,
uio=0xfffffe1839a54ab0,
flags=113246208) at /usr/src/sys/amd64/amd64/mem.c:101
101 error = uiomove((void *)PHYS_TO_DMAP(v),
(int)c, uio);
(kgdb) list
96 if (dev2unit(dev) == CDEV_MINOR_MEM) {
97 v = uio->uio_offset;
98 kmemphys:
99 o = v & PAGE_MASK;
100 c = min(uio->uio_resid, (u_int)(PAGE_SIZE - o));
101 error = uiomove((void *)PHYS_TO_DMAP(v),
(int)c, uio);
102 continue;
103 }
104 else if (dev2unit(dev) == CDEV_MINOR_KMEM) {
105 v = uio->uio_offset;
(kgdb) p *v
$4 = 0
(kgdb) p *c
$5 = 0
(kgdb) p *uio
$6 = {uio_iov = 0xfffffe1839a54aa0, uio_iovcnt = 1, uio_offset =
-2184830705664,
uio_resid = 113246208, uio_segflg = UIO_USERSPACE, uio_rw = UIO_READ,
uio_td = 0xfffff80e4edb8490}
(kgdb) down
#16 0xffffffff8099c2f5 in uiomove_faultflag (cp=<value optimized out>,
n=<value optimized out>, uio=0xfffffe1839a54ab0, nofault=<value optimized
out>)
at /usr/src/sys/kern/subr_uio.c:192
192 error = copyout(cp, iov->iov_base, cnt);
(kgdb) list
187 switch (uio->uio_segflg) {
188
189 case UIO_USERSPACE:
190 maybe_yield();
191 if (uio->uio_rw == UIO_READ)
192 error = copyout(cp, iov->iov_base, cnt);
193 else
194 error = copyin(iov->iov_base, cp, cnt);
195 if (error)
196 goto out;
(kgdb) p *cp
Attempt to dereference a generic pointer.
(kgdb) p cp
$7 = <value optimized out>
(kgdb) down
#15 0xffffffff80d97bab in copyout () at /usr/src/sys/amd64/amd64/support.S:246
246 cld
Current language: auto; currently asm
(kgdb) list
241 xchgq %rdi,%rsi
242 /* bcopy(%rsi, %rdi, %rdx) */
243 movq %rdx,%rcx
244
245 shrq $3,%rcx
246 cld
247 rep
248 movsq
249 movb %dl,%cl
250 andb $7,%cl
(kgdb) down
#14 0xffffffff80d7e6e2 in calltrap () at
/usr/src/sys/amd64/amd64/exception.S:231
231 call trap
(kgdb) list
226 #endif
227 .globl calltrap
228 .type calltrap,@function
229 calltrap:
230 movq %rsp,%rdi
231 call trap
232 MEXITCOUNT
233 jmp doreti /* Handle any pending ASTs */
234
235 /*
(kgdb) quit
Script done on Sun Mar 9 16:46:04 2014
Glen
pgpuXXCLkKdOm.pgp
Description: PGP signature
