[split into mult-parts]
I now have a core dump file for the crashes after some tweaking to get
this to work (starting savecore sooner + changing dumpadm to "swap"
instead of dedicated).
I can provide more info out of the core dump if someone provides the
commands to execute, etc:
This is an excerpt of what I saw on my console after coming back to my
session, followed by the kernel adb commands that I noticed by googling
around for common adb parameters.
I would really appreciate if someone could look into this asap. Let me
know if there is more info needed from ipf commands, etc
=======================================================================
[EMAIL PROTECTED] w
2:17am up 1 day(s), 16:36, 1 user, load average: 0.01, 0.03, 0.05
User tty login@ idle JCPU PCPU what
root console Mon 3am w
[EMAIL PROTECTED]
panic[cpu1]/thread=2a10008bd20: mutex_enter: bad mutex, lp=3000353c2c8
owner=2a100091d20 thread=2a10008bd20
000002a10008b5a0 unix:mutex_panic+5c (10415fb0, 3000353c2c8, 0,
300035ac43e, 2a100497a00, ffbefa58)
%l0-3: 00000000100072a4 000003000353c2c8 0000030003002de0
0000000010036054
%l4-7: 0000000010435958 0000000000000000 0000030002bd2320
000002a100497ba0
000002a10008b650 ipf:fr_derefrule+18 (2a10008b7d8, ffffffffffffffff,
ffffffffffffffff, 1041c380, 2a100497a00, 0)
%l0-3: 000000001035bdb0 0000000000000014 0000000000000038
00000000426dbaad
%l4-7: 0000000000000014 0000000000000001 0000000000000000
00000000ffbef9c0
000002a10008b720 ipf:fr_fragdelete+2c (300001c4368, 78018080,
2a10008bd24, 2a10008bd24, 10424d10, 0)
%l0-3: 0000000000000000 0000000000000002 0000000000000000
0000000000000000
%l4-7: 0000000000000000 0000000000000000 0000000000000000
0000000000000000
000002a10008b7f0 ipf:fr_fragexpire+98 (78128620, 78128620, 63,
ffffffffffffffff, 0, 0)
%l0-3: 000000001041d440 0000000000000116 00000300066d66c8
0000000000000000
%l4-7: 0000000000000000 000000000007bca0 000000000000004c
000000000003d800
000002a10008b8d0 ipf:fr_slowtimer+c4 (0, 2a10008bd20, 20, 10424d30, 16,
0)
%l0-3: 0000000010163514 000000000007bca7 00000000ffbef670
000002a10000fd20
%l4-7: 0000000000000001 0000000000020000 0000000000000600
0000000000000000
000002a10008b990 genunix:callout_execute+90 (bffffffffbb48d79, 1,
300001fb038, e7a2c2, 300001fa038, 0)
%l0-3: 00000000780e2738 8000000000000000 0000000000000006
00000300001fb648
%l4-7: 0000000000e7a2c2 00000300001fa000 0000030002c40f38
000002a10008b9a0
000002a10008ba40 genunix:taskq_thread+18c (30000a83d18, 0, 10424d30,
10000, 30000a83d4a, 30000a83d70)
%l0-3: 00000000100786c0 0000030000a83d48 0000030000a83d40
0000030000a83d18
%l4-7: 0000030000a83d38 0000030000a81ea8 00000300000678f0
0000000000000002
syncing file systems... 2 1 done
dumping to /dev/md/dsk/d20, offset 859111424
100% done: 24022 pages dumped, compression ratio 5.16, dump succeeded
rebooting...
....
....
....
....
syslog-ng service starting.
binding fd 3, inetaddr: 0.0.0.0, port: 514
binding fd 5, inetaddr: 0.0.0.0, port: 514
connecting fd 8 to inetaddr 10.207.9.3, port 514
connecting fd 9 to inetaddr 10.206.9.4, port 514
Apr 26 03:53:09 netlog1 unix: [ID 836849 kern.notice]
Apr 26 03:53:09 netlog1 panic[cpu1]/thread=2a10008bd20:
Apr 26 03:53:10 netlog1 unix: [ID 103648 kern.notice] mutex_enter: bad
mutex, lp=3000353c2c8 owner=2a100091d20 thread=2a10008bd20
Apr 26 03:53:10 netlog1 unix: [ID 100000 kern.notice]
Apr 26 03:53:10 netlog1 genunix: [ID 723222 kern.notice]
000002a10008b5a0 unix:mutex_panic+5c (10415fb0, 3000353c2c8, 0,
300035ac43e, 2a100497a00, ffbefa58)
Apr 26 03:53:10 netlog1 genunix: [ID 179002 kern.notice] %l0-3:
00000000100072a4 000003000353c2c8 0000030003002de0 0000000010036054
Apr 26 03:53:10 netlog1 %l4-7: 0000000010435958 0000000000000000
0000030002bd2320 000002a100497ba0
Apr 26 03:53:10 netlog1 genunix: [ID 723222 kern.notice]
000002a10008b650 ipf:fr_derefrule+18 (2a10008b7d8, ffffffffffffffff,
ffffffffffffffff, 1041c380, 2a100497a00, 0)
Apr 26 03:53:10 netlog1 genunix: [ID 179002 kern.notice] %l0-3:
000000001035bdb0 0000000000000014 0000000000000038 00000000426dbaad
Apr 26 03:53:10 netlog1 %l4-7: 0000000000000014 0000000000000001
0000000000000000 00000000ffbef9c0
Apr 26 03:53:10 netlog1 genunix: [ID 723222 kern.notice]
000002a10008b720 ipf:fr_fragdelete+2c (300001c4368, 78018080,
2a10008bd24, 2a10008bd24, 10424d10, 0)
Apr 26 03:53:10 netlog1 genunix: [ID 179002 kern.notice] %l0-3:
0000000000000000 0000000000000002 0000000000000000 0000000000000000
Apr 26 03:53:10 netlog1 %l4-7: 0000000000000000 0000000000000000
0000000000000000 0000000000000000
Apr 26 03:53:10 netlog1 genunix: [ID 723222 kern.notice]
000002a10008b7f0 ipf:fr_fragexpire+98 (78128620, 78128620, 63,
ffffffffffffffff, 0, 0)
Apr 26 03:53:10 netlog1 genunix: [ID 179002 kern.notice] %l0-3:
000000001041d440 0000000000000116 00000300066d66c8 0000000000000000
Apr 26 03:53:10 netlog1 %l4-7: 0000000000000000 000000000007bca0
000000000000004c 000000000003d800
Apr 26 03:53:10 netlog1 genunix: [ID 723222 kern.notice]
000002a10008b8d0 ipf:fr_slowtimer+c4 (0, 2a10008bd20, 20, 10424d30, 16,
0)
Apr 26 03:53:10 netlog1 genunix: [ID 179002 kern.notice] %l0-3:
0000000010163514 000000000007bca7 00000000ffbef670 000002a10000fd20
Apr 26 03:53:10 netlog1 %l4-7: 0000000000000001 0000000000020000
0000000000000600 0000000000000000
Apr 26 03:53:10 netlog1 genunix: [ID 723222 kern.notice]
000002a10008b990 genunix:callout_execute+90 (bffffffffbb48d79, 1,
300001fb038, e7a2c2, 300001fa038, 0)
Apr 26 03:53:10 netlog1 genunix: [ID 179002 kern.notice] %l0-3:
00000000780e2738 8000000000000000 0000000000000006 00000300001fb648
Apr 26 03:53:10 netlog1 %l4-7: 0000000000e7a2c2 00000300001fa000
0000030002c40f38 000002a10008b9a0
Apr 26 03:53:10 netlog1 genunix: [ID 723222 kern.notice]
000002a10008ba40 genunix:taskq_thread+18c (30000a83d18, 0, 10424d30,
10000, 30000a83d4a, 30000a83d70)
Apr 26 03:53:10 netlog1 genunix: [ID 179002 kern.notice] %l0-3:
00000000100786c0 0000030000a83d48 0000030000a83d40 0000030000a83d18
Apr 26 03:53:10 netlog1 %l4-7: 0000030000a83d38 0000030000a81ea8
00000300000678f0 0000000000000002
Apr 26 03:53:10 netlog1 unix: [ID 100000 kern.notice]
Apr 26 03:53:10 netlog1 genunix: [ID 672855 kern.notice] syncing file
systems...
Apr 26 03:53:10 netlog1 genunix: [ID 733762 kern.notice] 2
Apr 26 03:53:10 netlog1 genunix: [ID 733762 kern.notice] 1
Apr 26 03:53:10 netlog1 genunix: [ID 904073 kern.notice] done
Apr 26 03:53:10 netlog1 genunix: [ID 353387 kern.notice] dumping to
/dev/md/dsk/d20, offset 859111424
Apr 26 03:53:10 netlog1 genunix: [ID 409368 kern.notice] 100% done:
24022 pages dumped, compression ratio 5.16,
Apr 26 03:53:10 netlog1 genunix: [ID 851671 kern.notice] dump succeeded
Apr 26 03:53:10 netlog1 genunix: [ID 540533 kern.notice] SunOS Release
5.8 Version Generic_117350-20 64-bit
Apr 26 03:53:10 netlog1 genunix: [ID 913632 kern.notice] Copyright
1983-2003 Sun Microsystems, Inc. All rights reserved.
Apr 26 03:53:10 netlog1 genunix: [ID 370176 kern.warning] WARNING:
forceload of misc/md_trans failed
Apr 26 03:53:10 netlog1 genunix: [ID 370176 kern.warning] WARNING:
forceload of misc/md_raid failed
Apr 26 03:53:10 netlog1 genunix: [ID 370176 kern.warning] WARNING:
forceload of misc/md_hotspares failed
Apr 26 03:53:10 netlog1 genunix: [ID 370176 kern.warning] WARNING:
forceload of misc/md_sp failed
Apr 26 03:53:10 netlog1 rmclomv: [ID 758372 kern.notice] Hardware
watchdog enabled
Apr 26 03:53:10 netlog1 krtld: [ID 151736 kern.notice]
/kernel/drv/sparcv9/pfil symbol
Apr 26 03:53:10 netlog1 krtld: [ID 197480 kern.notice] miocpullup
multiply defined
Apr 26 03:52:48 netlog1 savecore: [ID 570001 auth.error] reboot after
panic: mutex_enter: bad mutex, lp=3000353c2c8 owner=2a100091d20
thread=2a10008bd20
Apr 26 03:52:48 netlog1 savecore: [ID 570001 auth.error] reboot after
panic: mutex_enter: bad mutex, lp=3000353c2c8 owner=2a100091d20
thread=2a10008bd20
Starting mysqld daemon with databases from /var/log/netlog/netlog1/mysql
starting syslog-ng2mysql...
volume management starting.
Starting mdlogd ...
The system is ready.
=======================================================================
[EMAIL PROTECTED] echo '$<msgbuf' | adb -k unix.0 vmcore.0
physmem 3dfeb
30001599ee3: pcisch2 is /[EMAIL PROTECTED],600000
30001599ac3: pcisch0 at root: SAFARI 0x1f 0x700000
300015996a3: pcisch0 is /[EMAIL PROTECTED],700000
30001599283: pcisch1 at root: SAFARI 0x1e 0x600000
30001598e63: pcisch1 is /[EMAIL PROTECTED],600000
30001598a43: pcisch3 at root: SAFARI 0x1d 0x700000
30001598623: pcisch3 is /[EMAIL PROTECTED],700000
30001598200: /[EMAIL PROTECTED],600000/[EMAIL PROTECTED] (glm0):
glm0 supports power management.
30001597da0: /[EMAIL PROTECTED],600000/[EMAIL PROTECTED] (glm0):
Rev. 0 Symbios 53c1010-33/66 found.
30001597982: PCI-device: [EMAIL PROTECTED], glm0
30001597563: glm0 is /[EMAIL PROTECTED],600000/[EMAIL PROTECTED]
30001597140: /[EMAIL PROTECTED],600000/[EMAIL PROTECTED],1 (glm1):
glm1 supports power management.
30001596d20: /[EMAIL PROTECTED],600000/[EMAIL PROTECTED],1 (glm1):
Rev. 0 Symbios 53c1010-33/66 found.
30001596902: PCI-device: [EMAIL PROTECTED],1, glm1
300015964e3: glm1 is /[EMAIL PROTECTED],600000/[EMAIL PROTECTED],1
300015960c0: sd0 at glm0: target 0 lun 0
3000157bc23: sd0 is /[EMAIL PROTECTED],600000/[EMAIL PROTECTED]/[EMAIL
PROTECTED],0
3000157b800: <SUN36G cyl 24620 alt 2 hd 27 sec 107>
3000157b3e0: sd1 at glm0: target 1 lun 0
3000157afc3: sd1 is /[EMAIL PROTECTED],600000/[EMAIL PROTECTED]/[EMAIL
PROTECTED],0
3000157aba0: <SUN36G cyl 24620 alt 2 hd 27 sec 107>
3000157a787: root on /pseudo/[EMAIL PROTECTED]:0,10,blk fstype ufs
3000157a363: WARNING: forceload of misc/md_trans failed
30001631f03: WARNING: forceload of misc/md_raid failed
30001631ae3: WARNING: forceload of misc/md_hotspares failed
300016316c3: WARNING: forceload of misc/md_sp failed
300016312a2: PCI-device: [EMAIL PROTECTED], ebus0
30001630e83: mc-us3i0 at root: SAFARI 0x0 0x0 ...
30001630a63: mc-us3i0 is /[EMAIL PROTECTED],0
30001630643: mc-us3i1 at root: SAFARI 0x1 0x0 ...
30001630223: mc-us3i1 is /[EMAIL PROTECTED],0
3000160bd80: power0 at ebus0: offset 0,800
3000160bac3: power0 is /[EMAIL PROTECTED],600000/[EMAIL PROTECTED]/[EMAIL
PROTECTED],800
3000160b6a0: rmc_comm0 at ebus0: offset 0,3e8
3000160b3e3: rmc_comm0 is /[EMAIL PROTECTED],600000/[EMAIL PROTECTED]/[EMAIL
PROTECTED],3e8
3000160b122: PCI-device: [EMAIL PROTECTED], pmubus0
3000160ae62: PCI-device: [EMAIL PROTECTED], pmugpio0
3000160aba3: pmugpio0 is /[EMAIL PROTECTED],600000/[EMAIL PROTECTED]/[EMAIL
PROTECTED]
3000160a8e2: pseudo-device: rmclomv0
3000160a623: rmclomv0 is /pseudo/[EMAIL PROTECTED]
3000160a363: Hardware watchdog enabled
3000160a0a0: su0 at ebus0: offset 0,3f8
30001843da3: su0 is /[EMAIL PROTECTED],600000/[EMAIL PROTECTED]/[EMAIL
PROTECTED],3f8
30001843ae0: su1 at ebus0: offset 0,2e8
30001843823: su1 is /[EMAIL PROTECTED],600000/[EMAIL PROTECTED]/[EMAIL
PROTECTED],2e8
30001843560: cpu0: UltraSPARC-IIIi (portid 0 impl 0x16 ver 0x24 clock
1002 MH
z)
300018432a0: cpu1: UltraSPARC-IIIi (portid 1 impl 0x16 ver 0x24 clock
1002 MH
z)
30001842fe0: cpu 1 initialization complete - online
30001842d1f: bge0: BCM570x driver v0.22: type "ether" mac address
00:03:ba:46
:11:4d
30001842a62: PCI-device: [EMAIL PROTECTED], bge0
300018427a3: bge0 is /[EMAIL PROTECTED],700000/[EMAIL PROTECTED]
300018424df: bge1: BCM570x driver v0.22: type "ether" mac address
00:03:ba:46
:11:4e
30001842222: PCI-device: [EMAIL PROTECTED],1, bge1
3000195bee3: bge1 is /[EMAIL PROTECTED],700000/[EMAIL PROTECTED],1
3000195bc1f: bge2: BCM570x driver v0.22: type "ether" mac address
00:03:ba:46
:11:4f
3000195b962: PCI-device: [EMAIL PROTECTED], bge2
3000195b6a3: bge2 is /[EMAIL PROTECTED],700000/[EMAIL PROTECTED]
3000195b3df: bge3: BCM570x driver v0.22: type "ether" mac address
00:03:ba:46
:11:50
3000195b122: PCI-device: [EMAIL PROTECTED],1, bge3
3000195ae63: bge3 is /[EMAIL PROTECTED],700000/[EMAIL PROTECTED],1
3000195aba1: /kernel/drv/sparcv9/pfil symbol
3000195a8e1: miocpullup multiply defined
30002885d9f: NOTICE: bge0: link up 100Mbps Full-Duplex (initialised)
3000288529f: NOTICE: bge1: link up 100Mbps Full-Duplex (initialised)
30002884e83: dump on /dev/md/dsk/d20 size 4096 MB
30002884a62: pseudo-device: devinfo0
300028847a3: devinfo0 is /pseudo/[EMAIL PROTECTED]
30002885822: PCI-device: [EMAIL PROTECTED], ohci0
30002885563: ohci0 is /[EMAIL PROTECTED],600000/[EMAIL PROTECTED]
30002884222: pseudo-device: fssnap0
30002c15c23: fssnap0 is /pseudo/[EMAIL PROTECTED]
30002c15ee2: pseudo-device: winlock0
300028840c3: winlock0 is /pseudo/[EMAIL PROTECTED]
30002c15962: pseudo-device: lockstat0
30002c153e3: lockstat0 is /pseudo/[EMAIL PROTECTED]
30002c15122: pseudo-device: vol0
30002c14e63: vol0 is /pseudo/[EMAIL PROTECTED]
30002c14d02: pseudo-device: llc10
30002c148e3: llc10 is /pseudo/[EMAIL PROTECTED]
30002c14622: pseudo-device: lofi0
30002c14363: lofi0 is /pseudo/[EMAIL PROTECTED]
30002c140a2: pseudo-device: fcp0
30002c91da3: fcp0 is /pseudo/[EMAIL PROTECTED]
30002c91982: pseudo-device: rmcadm0
30002c916c3: rmcadm0 is /pseudo/[EMAIL PROTECTED]
30002c91c3f: IP Filter: v4.1.8, running.
30002c91142: pseudo-device: twtchc0
30002c90e83: twtchc0 is /pseudo/[EMAIL PROTECTED]
30002c9079f: IP Filter: v4.1.8 detached.
30002c9021f: IP Filter: v4.1.8, running.
3000343b6a3: dump on /dev/dsk/c1t0d0s1 size 4096 MB
3000008dda2: pseudo-device: devinfo0
3000343bac3: devinfo0 is /pseudo/[EMAIL PROTECTED]
30001596e83: dump on /dev/md/dsk/d20 size 4096 MB
30001598780:
panic[cpu1]/thread=2a10008bd20:
3000157b540: mutex_enter: bad mutex, lp=3000353c2c8 owner=2a100091d20
thread=
2a10008bd20
3000157bd80:
30002884d23: 000002a10008b5a0 unix:mutex_panic+5c (10415fb0,
3000353c2c8, 0,
300035ac43e, 2a100497a00, ffbefa58)
3000195afc3: %l0-3: 00000000100072a4 000003000353c2c8
0000030003002de0 0000
000010036054
%l4-7: 0000000010435958 0000000000000000 0000030002bd2320
000002a100497ba0
3000343b283: 000002a10008b650 ipf:fr_derefrule+18 (2a10008b7d8,
fffffffffffff
fff, ffffffffffffffff, 1041c380, 2a100497a00, 0)
30002c14203: %l0-3: 000000001035bdb0 0000000000000014
0000000000000038 0000
0000426dbaad
%l4-7: 0000000000000014 0000000000000001 0000000000000000
00000000ffbef9c0
30001843983: 000002a10008b720 ipf:fr_fragdelete+2c (300001c4368,
78018080, 2a
10008bd24, 2a10008bd24, 10424d10, 0)
30002c14a43: %l0-3: 0000000000000000 0000000000000002
0000000000000000 0000
000000000000
%l4-7: 0000000000000000 0000000000000000 0000000000000000
0000000000000000
30001631823: 000002a10008b7f0 ipf:fr_fragexpire+98 (78128620,
78128620, 63, f
fffffffffffffff, 0, 0)
30001596a63: %l0-3: 000000001041d440 0000000000000116
00000300066d66c8 0000
000000000000
%l4-7: 0000000000000000 000000000007bca0 000000000000004c
000000000003d800
30002884bc3: 000002a10008b8d0 ipf:fr_slowtimer+c4 (0, 2a10008bd20,
20, 10424d
30, 16, 0)
30002c912a3: %l0-3: 0000000010163514 000000000007bca7
00000000ffbef670 0000
02a10000fd20
%l4-7: 0000000000000001 0000000000020000 0000000000000600
0000000000000000
30001631c43: 000002a10008b990 genunix:callout_execute+90
(bffffffffbb48d79, 1
, 300001fb038, e7a2c2, 300001fa038, 0)
30002c91403: %l0-3: 00000000780e2738 8000000000000000
0000000000000006 0000
0300001fb648
%l4-7: 0000000000e7a2c2 00000300001fa000 0000030002c40f38
000002a10008b9a0
30002c15803: 000002a10008ba40 genunix:taskq_thread+18c (30000a83d18,
0, 10424
d30, 10000, 30000a83d4a, 30000a83d70)
30002c90fe3: %l0-3: 00000000100786c0 0000030000a83d48
0000030000a83d40 0000
030000a83d18
%l4-7: 0000030000a83d38 0000030000a81ea8 00000300000678f0
0000000000000002
3000195b800:
3000157a0a3: syncing file systems...
30002c144c3: 2
3000195aa43: 1
3000008c4e3: done
30002c91563: dumping to /dev/md/dsk/d20, offset 859111424
[EMAIL PROTECTED]
