msgbuf is below: NOTICE: mppLnx_remove_proxyRequest_from_list() MPPLNX_QUEUE_QUEUED_LIST LockAddress:70411de0
panic[cpu2]/thread=2a100359cc0: BAD TRAP: type=31 rp=2a100359590 addr=0 mmu_fsr=0 occurred in module "vhba" due to a NULL pointer dereference sched: trap type = 0x31 pid=0, pc=0x131fd94, sp=0x2a100358e31, tstate=0x4400001605, context=0x0 g1-g7: 1856000, 205e, 2000, 3b, 60002e78ac8, 0, 2a100359cc0 000002a1003592b0 unix:die+78 (31, 2a100359590, 0, 0, 2a100359370, 1076000) %l0-3: 0000000000001fff 0000000000000031 0000000001000000 0000000000002000 %l4-7: 000000000181a1d8 000000000181a000 0000000000000000 00000000da766000 000002a100359390 unix:trap+9d4 (2a100359590, 10000, 1fff, 5, 0, 1) %l0-3: 0000000000000000 00000000018364c0 0000000000000031 0000000000000000 %l4-7: ffffffffffffe000 0000000000000000 0000000000000001 0000000000000005 000002a1003594e0 unix:ktl0+48 (70411de0, 0, 70400, 1, 40, 0) %l0-3: 0000000000000006 0000000000001400 0000004400001605 000000000101aa04 %l4-7: 000000000000000a 00000000018563d4 0000000000000000 000002a100359590 [b]000002a100359630 vhba:mppLnx_remove_proxyRequest_from_list+250 (60004fd7e10, 0, 1320000, 1320, 1000, 7045a000)[/b] %l0-3: 0000000000070411 0000000000070400 0000000070411000 0000000000070411 %l4-7: 0000000000070400 0000000001324000 0000000000001324 0000000000001000 000002a1003596e0 vhba:mppLnx_failoverCmd_done+470 (60000270d80, 0, 60000270c50, 60004fd7e10, 0, 600002b6000) %l0-3: 0000000001323000 0000000000000000 0000000000001323 00000000000005c5 %l4-7: 00000600002b6020 00000300003bd8c0 0000000000000020 0000000000000028 000002a1003597b0 fcp:ssfcp_cmd_callback+64 (60000270dd8, 0, 1, 300000b5ef8, 60000270be8, 60000183700) %l0-3: 0000000000000002 0000060000209000 0000000001843dd8 0000000000000008 %l4-7: 0000000000000001 0000000000000021 0000000000000000 00000000012cc400 000002a100359860 emlxs:emlxs_iodone+98 (60000270f78, 2a100359cc0, 60001bcf7f0, 18364c0, 16, 0) %l0-3: 00000000012c0000 0000060000270dd8 0000060000271028 00000300011f83a0 %l4-7: 0000000000003b01 0000000000000000 0000000000024110 00000000018a5800 000002a100359930 emlxs:emlxs_doneq_server+e8 (600000fe000, 0, 180c000, 3, 0, 0) %l0-3: 0000060000270f78 0000060000271028 0000000001843dd8 0000000000001242 %l4-7: ffffffffffffffff 000002a100351cc0 0000000000000002 0000000000000004 000002a100359a10 emlxs:emlxs_thread+dc (600000fe198, 0, 18364c0, 18364c0, 180c000, 0) %l0-3: 00000600000fe000 00000600000fe198 00000600000fe1d0 000000007bf397e0 %l4-7: 0000000001853af8 0000000000000000 000000000000028f 00000000018a5800 syncing file systems... 2 1 done dumping to /dev/dsk/c0t0d0s1, offset 1048510464, content: kernel $c gave me stack info: > $c mppLnx_remove_proxyRequest_from_list+0x264(60004fd7e10, 0, 1320000, 1320, 1000, 7045a000) mppLnx_failoverCmd_done+0x470(60000270d80, 0, 60000270c50, 60004fd7e10, 0, 600002b6000) ssfcp_cmd_callback+0x64(60000270dd8, 0, 1, 300000b5ef8, 60000270be8, 60000183700) emlxs_iodone+0x98(60000270f78, 2a100359cc0, 60001bcf7f0, 18364c0, 16, 0) emlxs_doneq_server+0xe8(600000fe000, 0, 180c000, 3, 0, 0) emlxs_thread+0xdc(600000fe198, 0, 18364c0, 18364c0, 180c000, 0) thread_start+4(600000fe198, 0, 0, 0, 0, 0) disassemble it > mppLnx_remove_proxyRequest_from_list+250::dis mppLnx_remove_proxyRequest_from_list+0x228: mov 0xa51, %o4 mppLnx_remove_proxyRequest_from_list+0x22c: sllx %l6, 0xc, %l5 mppLnx_remove_proxyRequest_from_list+0x230: or %l1, 0x11, %l0 mppLnx_remove_proxyRequest_from_list+0x234: sllx %l3, 0xc, %l2 mppLnx_remove_proxyRequest_from_list+0x238: add %l5, 0xbd8, %o1 mppLnx_remove_proxyRequest_from_list+0x23c: add %l2, 0xde0, %o2 mppLnx_remove_proxyRequest_from_list+0x240: call -0x1f2618 <cmn_err> mppLnx_remove_proxyRequest_from_list+0x244: mov 1, %o0 mppLnx_remove_proxyRequest_from_list+0x248: sllx %l0, 0xc, %o7 mppLnx_remove_proxyRequest_from_list+0x24c: add %o7, 0xde0, %o0 [b]mppLnx_remove_proxyRequest_from_list+0x250: call -0x2deb60 <mutex_enter>[/b]mppLnx_remove_proxyRequest_from_list+0x254: nop mppLnx_remove_proxyRequest_from_list+0x258: ldx [%i0 + 0x40], %o5 mppLnx_remove_proxyRequest_from_list+0x25c: sethi %hi(0x70400), %o2 mppLnx_remove_proxyRequest_from_list+0x260: ldx [%i0 + 0x48], %i1 mppLnx_remove_proxyRequest_from_list+0x264: stx %o5, [%i1] mppLnx_remove_proxyRequest_from_list+0x268: ldx [%i0 + 0x48], %o4 mppLnx_remove_proxyRequest_from_list+0x26c: ldx [%i0 + 0x40], %o3 mppLnx_remove_proxyRequest_from_list+0x270: stx %o4, [%o3 + 8] mppLnx_remove_proxyRequest_from_list+0x274: clrx [%i0 + 0x40] mppLnx_remove_proxyRequest_from_list+0x278: clrx [%i0 + 0x48] > mppLnx_remove_proxyRequest_from_list+0x264::dis mppLnx_remove_proxyRequest_from_list+0x23c: add %l2, 0xde0, %o2 mppLnx_remove_proxyRequest_from_list+0x240: call -0x1f2618 <cmn_err> mppLnx_remove_proxyRequest_from_list+0x244: mov 1, %o0 mppLnx_remove_proxyRequest_from_list+0x248: sllx %l0, 0xc, %o7 mppLnx_remove_proxyRequest_from_list+0x24c: add %o7, 0xde0, %o0 mppLnx_remove_proxyRequest_from_list+0x250: call -0x2deb60 <mutex_enter> mppLnx_remove_proxyRequest_from_list+0x254: nop mppLnx_remove_proxyRequest_from_list+0x258: ldx [%i0 + 0x40], %o5 mppLnx_remove_proxyRequest_from_list+0x25c: sethi %hi(0x70400), %o2 mppLnx_remove_proxyRequest_from_list+0x260: ldx [%i0 + 0x48], %i1 [b]mppLnx_remove_proxyRequest_from_list+0x264: stx %o5, [%i1][/b] mppLnx_remove_proxyRequest_from_list+0x268: ldx [%i0 + 0x48], %o4 mppLnx_remove_proxyRequest_from_list+0x26c: ldx [%i0 + 0x40], %o3 mppLnx_remove_proxyRequest_from_list+0x270: stx %o4, [%o3 + 8] mppLnx_remove_proxyRequest_from_list+0x274: clrx [%i0 + 0x40] mppLnx_remove_proxyRequest_from_list+0x278: clrx [%i0 + 0x48] mppLnx_remove_proxyRequest_from_list+0x27c: or %o2, 0x11, %i0 mppLnx_remove_proxyRequest_from_list+0x280: sllx %i0, 0xc, %o1 mppLnx_remove_proxyRequest_from_list+0x284: call -0x2deb14 <mutex_exit> mppLnx_remove_proxyRequest_from_list+0x288: add %o1, 0xde0, %o0 mppLnx_remove_proxyRequest_from_list+0x28c: ba +0x1f0 <mppLnx_remove_proxyRequest_from_list+0x47c> the register info : > $r %g0 = 0x0000000000000000 %l0 = 0x0000000000070411 %g1 = 0x0000000001856000 initargs+0x3c %l1 = 0x0000000000070400 %g2 = 0x000000000000205e %l2 = 0x0000000070411000 lockstat_probes+0x188 %g3 = 0x0000000000002000 %l3 = 0x0000000000070411 %g4 = 0x000000000000003b %l4 = 0x0000000000070400 %g5 = 0x0000060002e78ac8 %l5 = 0x0000000001324000 %g6 = 0x0000000000000000 %l6 = 0x0000000000001324 %g7 = 0x000002a100359cc0 %l7 = 0x0000000000001000 %o0 = 0x0000000070411de0 mppLnx_queuedProxyRequestQ %i0 = 0x0000060004fd7e10 %o1 = 0x0000000000000000 [b] %i1 = 0x0000000000000000[/b] %o2 = 0x0000000000070400 %i2 = 0x0000000001320000 %o3 = 0x0000000000000001 %i3 = 0x0000000000001320 %o4 = 0x0000000000000040 %i4 = 0x0000000000001000 %o5 = 0x0000000000000000 %i5 = 0x000000007045a000 %o6 = 0x000002a100358e31 %i6 = 0x000002a100358ee1 %o7 = 0x000000000131fd80 mppLnx_remove_proxyRequest_from_list+0x250 %i7 = 0x000000000131b8bc mppLnx_failoverCmd_done+0x470 %ccr = 0x44 xcc=nZvc icc=nZvc %fprs = 0x00 fef=0 du=0 dl=0 %asi = 0x00 %y = 0x0000000000000000 %pc = 0x000000000131fd94 mppLnx_remove_proxyRequest_from_list+0x264 %npc = 0x000000000131fd98 mppLnx_remove_proxyRequest_from_list+0x268 %sp = 0x000002a100358e31 unbiased=0x000002a100359630 %fp = 0x000002a100358ee1 %tick = 0x0000000000000000 %tba = 0x0000000000000000 %tt = 0x31 %tl = 0x0 %pil = 0x0 %pstate = 0x016 cle=0 tle=0 mm=TSO red=0 pef=1 am=0 priv=1 ie=1 ag=0 %cwp = 0x05 %cansave = 0x00 %canrestore = 0x00 %otherwin = 0x00 %wstate = 0x00 %cleanwin = 0x00 using the CC -S, we can see the code accroded to the above disassembles: ! 2641 ! "mppLnx_remove_proxyRequest_from_list() MPPLNX_QUEUE_QUEUED_LIS ! 2641 >T LockAddress:%p\n", &mppLnx_queuedProxyRequestQ.queueLock)); ! 2642 ! OSP_LockKmutexSaveIrq ( &mppLnx_queuedProxyRequestQ.queueLock, flags) ! 2642 >; /* 0x0220 2642 */ sethi %h44(mppLnx_queuedProxyRequestQ),%l1 /* 0x0224 2640 */ call cmn_err ! params = %o0 %o1 %o2 %o3 %o4 ! Result = /* 0x0228 */ or %g0,2641,%o4 /* 0x022c */ sllx %l6,12,%l5 /* 0x0230 2642 */ or %l1,%m44(mppLnx_queuedProxyRequestQ),%l0 /* 0x0234 2640 */ sllx %l3,12,%l2 /* 0x0238 */ add %l5,%l44(.L3398),%o1 /* 0x023c */ add %l2,%l44(mppLnx_queuedProxyRequestQ),%o2 /* 0x0240 */ call cmn_err ! params = %o0 %o1 %o2 ! Result = /* 0x0244 */ or %g0,1,%o0 /* 0x0248 2642 */ sllx %l0,12,%o7 /* 0x024c */ add %o7,%l44(mppLnx_queuedProxyRequestQ),%o0 [b]/* 0x0250 */ call mutex_enter ! params = %o0 ! Result =[/b] /* 0x0254 */ nop /* 0x0258 104 */ ldx [%i0+64],%o5 ! FILE mppLnx26_vhbalib.c ! 2643 ! OSP_RmvListEntry( &(pre->queued_list)); ! 2644 ! pre->queued_list.prev = NULL; ! 2645 ! pre->queued_list.next = NULL; ! 2647 ! OSP_UnlockKmutexStoreIrq ( &mppLnx_queuedProxyRequestQ.queueLock, fla ! 2647 >gs); /* 0x025c 2647 */ sethi %h44(mppLnx_queuedProxyRequestQ),%o2 /* 0x0260 104 */ ldx [%i0+72],%i1 [b]/* 0x0264 */ stx %o5,[%i1][/b] /* 0x0268 105 */ ldx [%i0+72],%o4 /* 0x026c */ ldx [%i0+64],%o3 /* 0x0270 */ stx %o4,[%o3+8] /* 0x0274 2644 */ stx %g0,[%i0+64] /* 0x0278 2645 */ stx %g0,[%i0+72] /* 0x027c 2647 */ or %o2,%m44(mppLnx_queuedProxyRequestQ),%i0 /* 0x0280 */ sllx %i0,12,%o1 /* 0x0284 */ call mutex_exit ! params = %o0 ! Result = /* 0x0288 */ add %o1,%l44(mppLnx_queuedProxyRequestQ),%o0 [b]the %i1 = 0x0000000000000000 is NULL[/b] but , what causes it ? it's so puzzle. OSP_LockKmutexSaveIrq is macro for mutex_enter: #define OSP_LockKmutexSaveIrq(lock, flags) \ mutex_enter(lock) the frist Input for the OSP_LockKmutexSaveIrq is a global Object: &mppLnx_queuedProxyRequestQ.queueLock, So it's not a NULL. > mppLnx_queuedProxyRequestQ::print { queueLock = { _opaque = [ 0x2a100359cc0 ] } list = { prev = 0x60004fd7a70 next = 0x60004fd7880 } queueType = 2 (MPPLNX_QUEUE_QUEUED_LIST) } > the queueLock which type is the kmutex_t is an object . I hope your idea about it ,Thanks. This message posted from opensolaris.org _______________________________________________ opensolaris-code mailing list opensolaris-code@opensolaris.org http://mail.opensolaris.org/mailman/listinfo/opensolaris-code