Ok, found it...there are 9 threads active: (gdb) info threads 9 process 43094 thread 0x2903 0x96ebf3ae in __semwait_signal () 8 process 43094 thread 0x2803 0x96ebf3ae in __semwait_signal () 7 process 43094 thread 0x2703 0x96ebf3ae in __semwait_signal () 6 process 43094 thread 0x2603 0x96eb8202 in semaphore_wait_trap () 5 process 43094 thread 0x2303 0x96ebf3ae in __semwait_signal () 4 process 43094 thread 0x2203 0x96ebf3ae in __semwait_signal () 3 process 43094 thread 0x2103 0x96efc796 in __wait4 () * 2 process 43094 thread 0x213 0x96ebf3ae in __semwait_signal () 1 process 43094 thread 0x717 0x96ebf3ae in __semwait_signal ()
Their backtraces are as follows: (gdb) thread 1 [Switching to thread 1 (process 43094 thread 0x717)] 0x96ebf3ae in __semwait_signal () (gdb) bt #0 0x96ebf3ae in __semwait_signal () #1 0x96f04398 in pthread_join$UNIX2003 () #2 0x0000668b in ContinueInNewThread0 () #3 0x0000440e in JLI_Launch () #4 0x0000d388 in main () (gdb) thread 2 [Switching to thread 2 (process 43094 thread 0x213)] 0x96ebf3ae in __semwait_signal () (gdb) bt #0 0x96ebf3ae in __semwait_signal () #1 0x96eea326 in _pthread_cond_wait () #2 0x96ee9d0d in pthread_cond_wait$UNIX2003 () #3 0x0136aea9 in os::PlatformEvent::park () #4 0x0134f32f in Monitor::IWait () #5 0x0134f5c4 in Monitor::wait () #6 0x01463070 in VMThread::execute () #7 0x0113e3a0 in GenCollectorPolicy::mem_allocate_work () #8 0x011cec88 in GenCollectedHeap::mem_allocate () #9 0x0142c27e in typeArrayKlass::allocate () #10 0x01362222 in oopFactory::new_typeArray () #11 0x010ef68f in Runtime1::new_type_array () #12 0x01ab508a in ?? () #13 0x01ae41f3 in ?? () #14 0x01ae3aa4 in ?? () #15 0x01aede5c in ?? () #16 0x01aef690 in ?? () #17 0x01af1684 in ?? () #18 0x01af1244 in ?? () #19 0x01af0cb0 in ?? () #20 0x01a4282f in ?? () #21 0x01a422cf in ?? () #22 0x01a421c7 in ?? () #23 0x01a421c7 in ?? () #24 0x01a421c7 in ?? () #25 0x01a421c7 in ?? () #26 0x01a421c7 in ?? () #27 0x01a421c7 in ?? () #28 0x01a421c7 in ?? () #29 0x01a421c7 in ?? () #30 0x01a421c7 in ?? () #31 0x01a41ff3 in ?? () #32 0x01a41ff3 in ?? () #33 0x01a3f2cc in ?? () #34 0x01222eb2 in JavaCalls::call_helper () #35 0x0136a241 in os::os_exception_wrapper () #36 0x01221a83 in JavaCalls::call () #37 0x011f67ac in instanceKlass::call_class_initializer_impl () #38 0x011f6a6e in instanceKlass::initialize_impl () #39 0x011f70c8 in instanceKlass::initialize () #40 0x013181f5 in LinkResolver::resolve_static_call () #41 0x01319e2b in LinkResolver::resolve_invokestatic () #42 0x01319f15 in LinkResolver::resolve_invoke () #43 0x0121b66b in InterpreterRuntime::resolve_invoke () #44 0x01a4f8cf in ?? () #45 0x01a421c7 in ?? () #46 0x01a421c7 in ?? () #47 0x01a421c7 in ?? () #48 0x01a41ff3 in ?? () #49 0x01a421c7 in ?? () #50 0x01a4216f in ?? () #51 0x01a4216f in ?? () #52 0x01a3f2cc in ?? () #53 0x01222eb2 in JavaCalls::call_helper () #54 0x0136a241 in os::os_exception_wrapper () #55 0x01221a83 in JavaCalls::call () #56 0x0122e6d8 in jni_invoke_static () #57 0x0124253d in jni_CallStaticVoidMethod () #58 0x00001f31 in JavaMain () #59 0x96ee9095 in _pthread_start () #60 0x96ee8f52 in thread_start () (gdb) thread 3 [Switching to thread 3 (process 43094 thread 0x2103)] 0x96efc796 in __wait4 () (gdb) bt #0 0x96efc796 in __wait4 () #1 0x96efc787 in waitpid$UNIX2003 () #2 0x0136a856 in os::fork_and_exec () #3 0x0145eae4 in VMError::show_message_box () #4 0x0145e87d in VMError::report_and_die () #5 0x01370b5e in JVM_handle_bsd_signal () #6 0x0136a031 in signalHandler () #7 <signal handler called> #8 0x011a43aa in frame::oops_interpreted_do () #9 0x011a4aa6 in frame::oops_do_internal () #10 0x0141f175 in JavaThread::oops_do () #11 0x0141f5ff in Threads::verify () #12 0x0142db1c in Universe::verify () #13 0x011d1506 in GenCollectedHeap::do_collection () #14 0x0113e5e3 in GenCollectorPolicy::satisfy_failed_allocation () #15 0x011cecc5 in GenCollectedHeap::satisfy_failed_allocation () #16 0x0145f23b in VM_GenCollectForAllocation::doit () #17 0x01464125 in VM_Operation::evaluate () #18 0x014627fd in VMThread::evaluate_operation () #19 0x0146346c in VMThread::loop () #20 0x0146375c in VMThread::run () #21 0x0136dab5 in java_start () #22 0x96ee9095 in _pthread_start () #23 0x96ee8f52 in thread_start () (gdb) thread 4 [Switching to thread 4 (process 43094 thread 0x2203)] 0x96ebf3ae in __semwait_signal () (gdb) bt #0 0x96ebf3ae in __semwait_signal () #1 0x96eea326 in _pthread_cond_wait () #2 0x96ee9d0d in pthread_cond_wait$UNIX2003 () #3 0x0136aea9 in os::PlatformEvent::park () #4 0x013ed57a in ObjectMonitor::wait () #5 0x013ed711 in ObjectSynchronizer::wait () #6 0x01293a5d in JVM_MonitorWait () #7 0x01a4971d in ?? () #8 0x01a41ff3 in ?? () #9 0x01a41ff3 in ?? () #10 0x01a3f2cc in ?? () #11 0x01222eb2 in JavaCalls::call_helper () #12 0x0136a241 in os::os_exception_wrapper () #13 0x012222d2 in JavaCalls::call_virtual () #14 0x0122250d in JavaCalls::call_virtual () #15 0x0128fefe in thread_entry () #16 0x01422671 in JavaThread::thread_main_inner () #17 0x0142275e in JavaThread::run () #18 0x0136dab5 in java_start () #19 0x96ee9095 in _pthread_start () #20 0x96ee8f52 in thread_start () (gdb) thread 5 [Switching to thread 5 (process 43094 thread 0x2303)] 0x96ebf3ae in __semwait_signal () (gdb) bt #0 0x96ebf3ae in __semwait_signal () #1 0x96eea326 in _pthread_cond_wait () #2 0x96ee9d0d in pthread_cond_wait$UNIX2003 () #3 0x0136aea9 in os::PlatformEvent::park () #4 0x013ed57a in ObjectMonitor::wait () #5 0x013ed711 in ObjectSynchronizer::wait () #6 0x01293a5d in JVM_MonitorWait () #7 0x01a4971d in ?? () #8 0x01a41ff3 in ?? () #9 0x01a421c7 in ?? () #10 0x01a421c7 in ?? () #11 0x01a3f2cc in ?? () #12 0x01222eb2 in JavaCalls::call_helper () #13 0x0136a241 in os::os_exception_wrapper () #14 0x012222d2 in JavaCalls::call_virtual () #15 0x0122250d in JavaCalls::call_virtual () #16 0x0128fefe in thread_entry () #17 0x01422671 in JavaThread::thread_main_inner () #18 0x0142275e in JavaThread::run () #19 0x0136dab5 in java_start () #20 0x96ee9095 in _pthread_start () #21 0x96ee8f52 in thread_start () (gdb) thread 6 [Switching to thread 6 (process 43094 thread 0x2603)] 0x96eb8202 in semaphore_wait_trap () (gdb) bt #0 0x96eb8202 in semaphore_wait_trap () #1 0x0136d8a0 in check_pending_signals () #2 0x0136d9c9 in os::signal_wait () #3 0x01367c85 in signal_thread_entry () #4 0x01422671 in JavaThread::thread_main_inner () #5 0x0142275e in JavaThread::run () #6 0x0136dab5 in java_start () #7 0x96ee9095 in _pthread_start () #8 0x96ee8f52 in thread_start () (gdb) thread 7 [Switching to thread 7 (process 43094 thread 0x2703)] 0x96ebf3ae in __semwait_signal () (gdb) bt #0 0x96ebf3ae in __semwait_signal () #1 0x96eea326 in _pthread_cond_wait () #2 0x96ee9d0d in pthread_cond_wait$UNIX2003 () #3 0x0136aea9 in os::PlatformEvent::park () #4 0x0134f32f in Monitor::IWait () #5 0x0134f5c4 in Monitor::wait () #6 0x01148fff in CompileQueue::get () #7 0x0114c91c in CompileBroker::compiler_thread_loop () #8 0x0141b154 in compiler_thread_entry () #9 0x01422671 in JavaThread::thread_main_inner () #10 0x0142275e in JavaThread::run () #11 0x0136dab5 in java_start () #12 0x96ee9095 in _pthread_start () #13 0x96ee8f52 in thread_start () (gdb) thread 8 [Switching to thread 8 (process 43094 thread 0x2803)] 0x96ebf3ae in __semwait_signal () (gdb) bt #0 0x96ebf3ae in __semwait_signal () #1 0x96eea326 in _pthread_cond_wait () #2 0x96ee9d0d in pthread_cond_wait$UNIX2003 () #3 0x0136aea9 in os::PlatformEvent::park () #4 0x0134f32f in Monitor::IWait () #5 0x0134f701 in Monitor::wait () #6 0x0131cfa4 in LowMemoryDetector::low_memory_detector_thread_entry () #7 0x01422671 in JavaThread::thread_main_inner () #8 0x0142275e in JavaThread::run () #9 0x0136dab5 in java_start () #10 0x96ee9095 in _pthread_start () #11 0x96ee8f52 in thread_start () (gdb) thread 9 [Switching to thread 9 (process 43094 thread 0x2903)] 0x96ebf3ae in __semwait_signal () (gdb) bt #0 0x96ebf3ae in __semwait_signal () #1 0x96eea326 in _pthread_cond_wait () #2 0x96f0f9f0 in pthread_cond_timedwait$UNIX2003 () #3 0x0136c019 in os::PlatformEvent::park () #4 0x0136f164 in os::sleep () #5 0x0142086f in WatcherThread::run () #6 0x0136dab5 in java_start () #7 0x96ee9095 in _pthread_start () #8 0x96ee8f52 in thread_start () On Tue, Aug 4, 2009 at 10:28 PM, Charles Oliver Nutter<head...@headius.com> wrote: > The ShowMessageBox option also gave me the option of launching gdb. > I'm not a gdb expert, but "bt" produced this: > > (gdb) bt > #0 0x96ebf3ae in __semwait_signal () > #1 0x96f04398 in pthread_join$UNIX2003 () > #2 0x0000668b in ContinueInNewThread0 () > #3 0x0000440e in JLI_Launch () > #4 0x0000d388 in main () > > Trying now to see if I can view stacks for other threads or something. > > On Tue, Aug 4, 2009 at 10:24 PM, Charles Oliver > Nutter<head...@headius.com> wrote: >> Here's the output with the verify flags on: >> >> ~/projects/jruby ➔ jruby -J-XX:+UnlockDiagnosticVMOptions >> -J-XX:+VerifyBeforeGC -J-XX:+VerifyAfterGC >> -J-Djruby.compile.invokedynamic -J-XX:+EnableInvokeDynamic >> bench/bench_fib_recursive.rb 100 >> [Verifying threads permgen tenured generation def new generation >> remset ref_proc syms strs zone dict hand C-heap ] >> VerifyBeforeGC:[Verifying threads permgen tenured generation def new >> generation remset ref_proc syms strs zone dict hand C-heap ] >> VerifyAfterGC:[Verifying threads permgen tenured generation def new >> generation remset ref_proc syms strs zone dict hand C-heap ] >> VerifyBeforeGC:[Verifying threads permgen tenured generation def new >> generation remset ref_proc syms strs zone dict hand C-heap ] >> VerifyAfterGC:[Verifying threads permgen tenured generation def new >> generation remset ref_proc syms strs zone dict hand C-heap ] >> VerifyBeforeGC:[Verifying threads permgen tenured generation def new >> generation remset ref_proc syms strs zone dict hand C-heap ] >> VerifyAfterGC:[Verifying threads permgen tenured generation def new >> generation remset ref_proc syms strs zone dict hand C-heap ] >> VerifyBeforeGC:[Verifying threads # >> >> The hs_err file is attached. I do not have pstack on OS X...do you >> know what the equivalent would be? >> >> On Tue, Aug 4, 2009 at 8:56 PM, <y.s.ramakris...@sun.com> wrote: >>> The crash is in GC. Together with yr conjecture that the crash happens >>> after a while when you expect the method to have been jitted, this might >>> imply that the jit is not correctly conveying to GC where the >>> references in jitted frames are or is not marking cards for the >>> generational collector correctly, or GC isn't looking at all the >>> slots it should be looking at or .... (But that is a long assumption.) >>> The funny part about "running long enough" is that the hs_err log >>> says the crash occurred after 0 seconds, so the VM is probably >>> not correctly printing the time it's been running. >>> >>> I'd suggest running the JVM with the options -XX:+VerifyBeforeGC and >>> -XX:+VerifyAfterGC >>> to perhaps catch the problem sooner or get closer to the root cause. >>> If you run with the additional JVM option -XX:+ShowMessageBoxOnError >>> you might be able to pstack the process and get some info. (Or >>> use the hs_err script to decode the stack trace from the hs_err log.) >>> >>> -- ramki >>> >>> On 08/04/09 18:07, Charles Oliver Nutter wrote: >>>> I can't seem to catch a break. My local build and Attila's build still >>>> both crash for me. There's got to be something about my environment >>>> that's different, eh? >>>> >>>> As before, interpreter runs fine. Anything that runs long enough to >>>> jit seems to crash. I've attached the latest log. >>>> >>>> On Tue, Aug 4, 2009 at 7:58 PM, Charles Oliver >>>> Nutter<head...@headius.com> wrote: >>>>> On Tue, Aug 4, 2009 at 4:17 PM, Attila Szegedi<szege...@gmail.com> wrote: >>>>>> I can now confirm that it indeed does work with jruby-1.4.0dev that >>>>>> indeed emits invokedynamic. >>>>>> >>>>>> The only catch is, it's very slow with invokedynamic, something that >>>>>> Christian also mentioned. >>>>> The fact that these numbers are so bad and even degrade shows me that >>>>> although it's compiling indy, it's not eliminating the handles (which >>>>> is true, the inlining and handle-walking stuff is not ready yet). So >>>>> it's got many layers of polymorphic calls, probably some argument >>>>> boxing and unboxing, and basically a ton more overhead than our >>>>> heavily tuned non-indy call path. I'm confident in the Hotspot guys :) >>>>> >>>>> Just wanted to make sure anyone watching didn't think this was the >>>>> best it would get... >>>>> >>>>>> MacBook-Pro-Ati:jruby-1.4.0dev aszegedi$ bin/jruby --server -J- >>>>>> Djruby.compile.invokedynamic=true -J-XX:+EnableInvokeDynamic bench/ >>>>>> bench_fib_recursive.rb 15 >>>>>> 2.993000 0.000000 2.993000 ( 2.915000) >>>>>> 2.720000 0.000000 2.720000 ( 2.720000) >>>>>> 2.773000 0.000000 2.773000 ( 2.773000) >>>>>> 2.712000 0.000000 2.712000 ( 2.712000) >>>>>> 3.057000 0.000000 3.057000 ( 3.057000) >>>>>> 3.183000 0.000000 3.183000 ( 3.184000) >>>>>> 3.167000 0.000000 3.167000 ( 3.167000) >>>>>> 3.145000 0.000000 3.145000 ( 3.145000) >>>>>> 3.249000 0.000000 3.249000 ( 3.249000) >>>>>> 3.160000 0.000000 3.160000 ( 3.160000) >>>>>> 3.104000 0.000000 3.104000 ( 3.104000) >>>>>> 3.152000 0.000000 3.152000 ( 3.153000) >>>>>> 3.172000 0.000000 3.172000 ( 3.171000) >>>>>> 3.111000 0.000000 3.111000 ( 3.111000) >>>>>> 3.201000 0.000000 3.201000 ( 3.201000) >>>>>> MacBook-Pro-Ati:jruby-1.4.0dev aszegedi$ bin/jruby --server bench/ >>>>>> bench_fib_recursive.rb 15 >>>>>> 0.523000 0.000000 0.523000 ( 0.481000) >>>>>> 0.292000 0.000000 0.292000 ( 0.291000) >>>>>> 0.279000 0.000000 0.279000 ( 0.278000) >>>>>> 0.283000 0.000000 0.283000 ( 0.283000) >>>>>> 0.284000 0.000000 0.284000 ( 0.284000) >>>>>> 0.292000 0.000000 0.292000 ( 0.292000) >>>>>> 0.289000 0.000000 0.289000 ( 0.289000) >>>>>> 0.286000 0.000000 0.286000 ( 0.287000) >>>>>> 0.285000 0.000000 0.285000 ( 0.285000) >>>>>> 0.285000 0.000000 0.285000 ( 0.285000) >>>>>> 0.289000 0.000000 0.289000 ( 0.290000) >>>>>> 0.282000 0.000000 0.282000 ( 0.282000) >>>>>> 0.290000 0.000000 0.290000 ( 0.290000) >>>>>> 0.293000 0.000000 0.293000 ( 0.293000) >>>>>> 0.285000 0.000000 0.285000 ( 0.284000) >>>>> >>>>> ------------------------------------------------------------------------ >>>>> >>>>> _______________________________________________ >>>>> mlvm-dev mailing list >>>>> mlvm-dev@openjdk.java.net >>>>> http://mail.openjdk.java.net/mailman/listinfo/mlvm-dev >>> >>> _______________________________________________ >>> mlvm-dev mailing list >>> mlvm-dev@openjdk.java.net >>> http://mail.openjdk.java.net/mailman/listinfo/mlvm-dev >>> >> > _______________________________________________ mlvm-dev mailing list mlvm-dev@openjdk.java.net http://mail.openjdk.java.net/mailman/listinfo/mlvm-dev
