Just to recap, the system is running but cannot fork anymore procs. This means I can't login. Cron does a make -j3 buildworld each night and it appears this is triggering it. Sys is uniprocessor. Please let me know if you need more info.
db> ps pid proc addr uid ppid pgrp flag stat wmesg wchan cmd 66984 c24efa80 cddf7000 0 1 1 8000000 norm[SLPQ thrd_sleep c0b740b4][SLP] init 66983 c24efc40 cddf8000 2 66982 66983 8000110 norm[SLPQ thrd_sleep c0b740b4][SLP] cron 66982 c24efe00 cddf9000 0 420 420 0000000 norm[SLPQ ppwait c24efe00][SLP] cron 66981 c24d5e00 cdca7000 2 66980 66981 8000110 norm[SLPQ thrd_sleep c0b740b4][SLP] cron 66980 c24d5c40 cdca6000 0 420 420 0000000 norm[SLPQ ppwait c24d5c40][SLP] cron 66979 c24d6000 cdca8000 2 66978 66979 8000110 norm[SLPQ thrd_sleep c0b740b4][SLP] cron 66978 c24c3540 cdb4a000 0 420 420 0000000 norm[SLPQ ppwait c24c3540][SLP] cron 66977 c24c3700 cdb4b000 2 66976 66977 8000110 norm[SLPQ thrd_sleep c0b740b4][SLP] cron 66976 c24c38c0 cdb4c000 0 420 420 0000000 norm[SLPQ ppwait c24c38c0][SLP] cron 66975 c24c3a80 cdb4d000 2 66973 66975 8000110 norm[SLPQ thrd_sleep c0b740b4][SLP] cron 66974 c24c3c40 cdb4e000 0 66972 66974 8000110 norm[SLPQ thrd_sleep c0b740b4][SLP] cron 66973 c24c3e00 cdb4f000 0 420 420 0000000 norm[SLPQ ppwait c24c3e00][SLP] cron 66972 c24d1000 cdc24000 0 420 420 0000000 norm[SLPQ ppwait c24d1000][SLP] cron 66971 c24d11c0 cdc25000 2 66970 66971 8000110 norm[SLPQ thrd_sleep c0b740b4][SLP] cron 66970 c24d1380 cdc98000 0 420 420 0000000 norm[SLPQ ppwait c24d1380][SLP] cron 66969 c24d1540 cdc99000 2 66968 66969 8000110 norm[SLPQ thrd_sleep c0b740b4][SLP] cron 66968 c24d1700 cdc9a000 0 420 420 0000000 norm[SLPQ ppwait c24d1700][SLP] cron 66967 c24d18c0 cdc9b000 2 66966 66967 8000110 norm[SLPQ thrd_sleep c0b740b4][SLP] cron 66966 c24d1a80 cdc9c000 0 420 420 0000000 norm[SLPQ ppwait c24d1a80][SLP] cron 66965 c24d1c40 cdc9d000 2 66964 66965 8000110 norm[SLPQ thrd_sleep c0b740b4][SLP] cron [... more pages of the same ...] 66780 c1a21000 cd37d000 0 420 420 0000000 norm[SLPQ ppwait c1a21000][SLP] cron 66779 c1a2cc40 cd2f8000 2 66778 66779 8000110 norm[SLPQ thrd_sleep c0b740b4][SLP] cron 66778 c1b62e00 cd3c7000 0 420 420 0000000 norm[SLPQ ppwait c1b62e00][SLP] cron 66777 c1a2c380 cd2f3000 2 66776 66777 8000110 norm[SLPQ thrd_sleep c0b740b4][SLP] cron 66776 c19d1700 cd334000 0 420 420 0000000 norm[SLPQ ppwait c19d1700][SLP] cron 66775 c1e62c40 cd45e000 0 66768 1431 0004000 norm[SLPQ thrd_sleep c0b740b4][SLP] as 66774 c1a21a80 cd3bc000 0 66768 1431 8000000 norm[SLPQ ufs c226168c][SLP] cc 66773 c19cda80 cd319000 0 66769 1431 0004000 norm[SLPQ ufs c220d1ec][SLP] as 66772 c19d1380 cd332000 0 66769 1431 0004000 norm[SLPQ thrd_sleep c0b740b4][SLP] cc1 66771 c1a2ca80 cd2f7000 0 66767 1431 8000000 norm[SLPQ thrd_sleep c0b740b4][SLP] cc 66770 c19cd8c0 cd318000 0 66767 1431 0004000 norm[SLPQ ufs c226168c][SLP] cc1 66769 c19271c0 cce49000 0 66766 1431 0004000 norm[SLPQ wait c19271c0][SLP] cc 66768 c1a2ce00 cd2f9000 0 66764 1431 0004000 norm[SLPQ wait c1a2ce00][SLP] cc 66767 c19d1540 cd333000 0 66765 1431 0004000 norm[SLPQ wait c19d1540][SLP] cc 66766 c1b62000 cd3bf000 0 66763 1431 0004000 norm[SLPQ wait c1b62000][SLP] sh 66765 c1e62e00 cd45f000 0 66763 1431 0004000 norm[SLPQ wait c1e62e00][SLP] sh 66764 c1a21c40 cd3bd000 0 66763 1431 0004000 norm[SLPQ wait c1a21c40][SLP] sh 66763 c1e62540 cd45a000 0 42503 1431 0004000 norm[CVQ select c03c4fc4][SLP] make 42503 c19d18c0 cd335000 0 42502 1431 0004000 norm[SLPQ wait c19d18c0][SLP] sh 42502 c1b621c0 cd3c0000 0 37343 1431 0004000 norm[CVQ select c03c4fc4][SLP] make 37343 c1927380 cce4a000 0 37337 1431 0004000 norm[SLPQ wait c1927380][SLP][SWAP] sh 37337 c19d1c40 cd337000 0 17464 1431 0004000 norm[CVQ select c03c4fc4][SLP] make 17464 c19cd700 cd317000 0 17459 1431 0004000 norm[SLPQ wait c19cd700][SLP][SWAP] sh 17459 c19cd380 cd309000 0 17458 1431 0004000 norm[CVQ select c03c4fc4][SLP] 17458 c1a21e00 cd3be000 0 1451 1431 0004000 norm[SLPQ wait c1a21e00][SLP][SWAP] sh 1451 c19cd1c0 cd308000 0 1450 1431 0004000 norm[CVQ select c03c4fc4][SLP] make 1450 c19d11c0 cd329000 0 1436 1431 0004000 norm[SLPQ wait c19d11c0][SLP][SWAP] sh 1436 c19d1000 cd328000 0 1433 1431 0004000 norm[CVQ select c03c4fc4][SLP] make 1433 c1a2c540 cd2f4000 0 1431 1431 0004000 norm[SLPQ wait c1a2c540][SLP][SWAP] sh 1431 c1924380 cce08000 0 1429 1431 0004000 norm[SLPQ wait c1924380][SLP][SWAP] sh 1429 c19cde00 cd327000 0 420 420 0000000 norm[SLPQ piperd c18aaa50][SLP] cron 420 c19241c0 cce07000 0 1 420 0000000 norm[SLPQ nanslp c03f9bd4][SLP] cron db> tr 66770 mi_switch(c1a2d410,50,c037b496,cc,1) at mi_switch+0x1e1 msleep(c226168c,c03f8010,50,c038b03e,0) at msleep+0x453 acquire(c226168c,1000040,700,101,104d2) at acquire+0xa7 lockmgr(c226168c,1030041,c22615c8,c1a2d410,cd2cab34) at lockmgr+0x16d vop_stdlock(cd2cab50,cd2cab70,c028902b,cd2cab50,c0382bd2) at vop_stdlock+0x2c ufs_vnoperate(cd2cab50,c0382bd2,815,cd2cab70,c0220731) at ufs_vnoperate+0x18 vn_lock(c22615c8,1030041,c1a2d410,815,18b) at vn_lock+0xeb vget(c22615c8,1020041,c1a2d410,432,18b) at vget+0xd6 vnode_pager_lock(c1f5e5b0,8230000,1,cd2cac60,cd2cac50) at vnode_pager_lock+0x95 vm_fault(c19c4b28,8230000,1,0,c1a2d410) at vm_fault+0x2cb trap_pfault(cd2cad48,1,8230ab0,299,8230ab0) at trap_pfault+0xef trap(2f,2f,2f,bfbff664,2d) at trap+0x287 calltrap() at calltrap+0x5 --- trap 0xc, eip = 0x8230ab0, esp = 0xbfbff640, ebp = 0xbfbff65c --- db> tr 66984 mi_switch(c24eba90,44,c037b496,cc,cdda6a38) at mi_switch+0x1e1 msleep(c0b740b4,c03f7928,44,c038c5d9,0) at msleep+0x453 acquire(c0b740b4,1000000,600,e3,105a8) at acquire+0xa7 lockmgr(c0b740b4,2,0,c24eba90,246) at lockmgr+0x378 _vm_map_lock(c0b74078,c038c47c,1d0,c02207e0,c24efae8) at _vm_map_lock+0x5b kmem_alloc_wait(c0b74078,11000,c03784b8,cb,c03fa420) at kmem_alloc_wait+0x38 kern_execve(c24eba90,80a188d,bfbffca0,bfbffe04,0) at kern_execve+0x1de execve(c24eba90,cdda6d10,c039575a,409,3) at execve+0x30 syscall(2f,2f,2f,bfbffdf8,0) at syscall+0x28e Xint0x80_syscall() at Xint0x80_syscall+0x1d --- syscall (59, FreeBSD ELF32, execve), eip = 0x80524af, esp = 0xbfbffbbc, ebp = 0xbfbffbd8 --- db> tr 66983 mi_switch(c24eb9c0,44,c037b496,cc,c03fc118) at mi_switch+0x1e1 msleep(c0b740b4,c03f7928,44,c038c5d9,0) at msleep+0x453 acquire(c0b740b4,1000000,600,e3,105a7) at acquire+0xa7 lockmgr(c0b740b4,2,0,c24eb9c0,246) at lockmgr+0x378 _vm_map_lock(c0b74078,c038c47c,1d0,c02207e0,c24efca8) at _vm_map_lock+0x5b kmem_alloc_wait(c0b74078,11000,c03784b8,cb,0) at kmem_alloc_wait+0x38 kern_execve(c24eb9c0,80590c6,bfbff6ec,8052160,0) at kern_execve+0x1de execve(c24eb9c0,cdda3d10,c039575a,409,3) at execve+0x30 syscall(2f,2f,2f,805213a,bfbff6ec) at syscall+0x28e Xint0x80_syscall() at Xint0x80_syscall+0x1d --- syscall (59, FreeBSD ELF32, execve), eip = 0x280c1343, esp = 0xbfbff6dc, ebp = 0xbfbff708 --- db> tr 66982 mi_switch(c24eb8f0,5c,c037b496,cc,c24f21b0) at mi_switch+0x1e1 msleep(c24efe00,c24efca8,5c,c0378708,0) at msleep+0x453 fork1(c24eb8f0,80000034,0,cdda0cd4,c24efe68) at fork1+0x11bb vfork(c24eb8f0,cdda0d10,c039575a,409,0) at vfork+0x52 syscall(2f,2f,2f,0,8052040) at syscall+0x28e Xint0x80_syscall() at Xint0x80_syscall+0x1d --- syscall (66, FreeBSD ELF32, vfork), eip = 0x2809e52c, esp = 0xbfbff710, ebp = 0xbfbffcd8 --- db> tr 66981 mi_switch(c24d28f0,44,c037b496,cc,c03fc118) at mi_switch+0x1e1 msleep(c0b740b4,c03f7928,44,c038c5d9,0) at msleep+0x453 acquire(c0b740b4,1000000,600,e3,105a5) at acquire+0xa7 lockmgr(c0b740b4,2,0,c24d28f0,246) at lockmgr+0x378 _vm_map_lock(c0b74078,c038c47c,1d0,c02207e0,c24d5e68) at _vm_map_lock+0x5b kmem_alloc_wait(c0b74078,11000,c03784b8,cb,0) at kmem_alloc_wait+0x38 kern_execve(c24d28f0,80590c6,bfbff6ec,8052160,0) at kern_execve+0x1de execve(c24d28f0,cdc49d10,c039575a,409,3) at execve+0x30 syscall(2f,2f,2f,805213a,bfbff6ec) at syscall+0x28e Xint0x80_syscall() at Xint0x80_syscall+0x1d --- syscall (59, FreeBSD ELF32, execve), eip = 0x280c1343, esp = 0xbfbff6dc, ebp = 0xbfbff708 --- db> tr 66980 mi_switch(c24d29c0,5c,c037b496,cc,c24d4360) at mi_switch+0x1e1 msleep(c24d5c40,c24d5e68,5c,c0378708,0) at msleep+0x453 fork1(c24d29c0,80000034,0,cdc4ccd4,c24d5ca8) at fork1+0x11bb vfork(c24d29c0,cdc4cd10,c039575a,409,0) at vfork+0x52 syscall(2f,2f,2f,0,8052040) at syscall+0x28e Xint0x80_syscall() at Xint0x80_syscall+0x1d --- syscall (66, FreeBSD ELF32, vfork), eip = 0x2809e52c, esp = 0xbfbff710, ebp = 0xbfbffcd8 --- db> tr 66772 mi_switch(c1926750,44,c037b496,cc,c03fc208) at mi_switch+0x1e1 msleep(c0b740b4,c03f7928,44,c038c5d9,0) at msleep+0x453 acquire(c0b740b4,1000000,600,e3,104d4) at acquire+0xa7 lockmgr(c0b740b4,2,0,c1926750,c1926750) at lockmgr+0x378 _vm_map_lock(c0b74078,c038c47c,1ed,c1b95800,c1926750) at _vm_map_lock+0x5b kmem_free_wakeup(c0b74078,c7f0a000,11000,0,0) at kmem_free_wakeup+0x2a kern_execve(c1926750,806b300,8073400,8070300,0) at kern_execve+0xd57 execve(c1926750,cce29d10,c039575a,409,3) at execve+0x30 syscall(2f,2f,2f,0,0) at syscall+0x28e Xint0x80_syscall() at Xint0x80_syscall+0x1d --- syscall (0, FreeBSD ELF32, nosys), eip = 0x80480c0, esp = 0xbfbff630, ebp = 0 --- To me, it looks like 66772 (in kmem_free_wakeup/vm_map_lock()) and the others (in kmem_alloc_wait/vm_map_lock()) are the problem. But I have no idea who has the lock. "show locks" just shows Giant locked by the intr handler for ddb. I can leave the box in ddb for a while if anyone needs more info. -Nate To Unsubscribe: send mail to [EMAIL PROTECTED] with "unsubscribe freebsd-current" in the body of the message
