[hadoop@hadoop1 bin]$ jstack 27737
2013-04-17 16:33:07
Full thread dump Java HotSpot(TM) 64-Bit Server VM (22.1-b02 mixed mode):
"Attach Listener" daemon prio=10 tid=0x00007f8f34006000 nid=0x625f waiting
on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"SIGTERM handler" daemon prio=10 tid=0x00007f8f34005800 nid=0x5281 waiting
for monitor entry [0x00007f8f57303000]
java.lang.Thread.State: BLOCKED (on object monitor)
at java.lang.Shutdown.exit(Shutdown.java:212)
- waiting to lock <0x000000061b178a08> (a java.lang.Class for
java.lang.Shutdown)
at java.lang.Terminator$1.handle(Terminator.java:52)
at sun.misc.Signal$1.run(Signal.java:212)
at java.lang.Thread.run(Thread.java:722)
"Thread-5" prio=10 tid=0x00007f8dbc00a000 nid=0x1901 in Object.wait()
[0x00007f8d4de9d000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Thread.join(Thread.java:1258)
- locked <0x000000061ae2ba58> (a java.lang.Thread)
at org.apache.hadoop.hbase.util.Threads.shutdown(Threads.java:94)
at org.apache.hadoop.hbase.util.Threads.shutdown(Threads.java:82)
at
org.apache.hadoop.hbase.regionserver.ShutdownHook$ShutdownHookThread.run(ShutdownHook.java:114)
at
org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54)
"SIGTERM handler" daemon prio=10 tid=0x00007f8f34004800 nid=0x1900 in
Object.wait() [0x00007f8d4efae000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Thread.join(Thread.java:1258)
- locked <0x000000061b09a740> (a
org.apache.hadoop.util.ShutdownHookManager$1)
at java.lang.Thread.join(Thread.java:1332)
at
java.lang.ApplicationShutdownHooks.runHooks(ApplicationShutdownHooks.java:106)
at
java.lang.ApplicationShutdownHooks$1.run(ApplicationShutdownHooks.java:46)
at java.lang.Shutdown.runHooks(Shutdown.java:123)
at java.lang.Shutdown.sequence(Shutdown.java:167)
at java.lang.Shutdown.exit(Shutdown.java:212)
- locked <0x000000061b178a08> (a java.lang.Class for
java.lang.Shutdown)
at java.lang.Terminator$1.handle(Terminator.java:52)
at sun.misc.Signal$1.run(Signal.java:212)
at java.lang.Thread.run(Thread.java:722)
"RS_CLOSE_ROOT-hadoop1,60020,1365731099431-0" prio=10
tid=0x00007f8eec346000 nid=0x4f94 waiting on condition [0x00007f8f57bf6000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000061b37e050> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
"RS_CLOSE_REGION-hadoop1,60020,1365731099431-2" prio=10
tid=0x00007f8d5c005800 nid=0x4ba8 waiting on condition [0x00007f8f57202000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000061b381df8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
"RS_CLOSE_REGION-hadoop1,60020,1365731099431-1" prio=10
tid=0x00007f8d6c005800 nid=0x5322 waiting on condition [0x00007f8d4edac000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000061b381df8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
"RS_CLOSE_REGION-hadoop1,60020,1365731099431-0" prio=10
tid=0x00007f8d70006800 nid=0x1dc7 waiting on condition [0x00007f8d4e6a5000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000061b381df8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
"StoreFileSplitter-8" prio=10 tid=0x00007f8dd4075000 nid=0xac7 waiting on
condition [0x00007f8d4db9a000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000066b435850> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
"StoreFileSplitter-7" prio=10 tid=0x00007f8dd4041000 nid=0xac6 waiting on
condition [0x00007f8d4e9a8000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000066b435850> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
"StoreFileSplitter-6" prio=10 tid=0x00007f8dd406a000 nid=0xac5 waiting on
condition [0x00007f8d4e8a7000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000066b435850> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
"StoreFileSplitter-5" prio=10 tid=0x00007f8dd4069000 nid=0xac4 waiting on
condition [0x00007f8f57ffe000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000066b435850> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
"StoreFileSplitter-4" prio=10 tid=0x00007f8dd4015000 nid=0xac3 waiting on
condition [0x00007f8d4e1a0000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000066b435850> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
"StoreFileSplitter-3" prio=10 tid=0x00007f8dd4043800 nid=0xac2 waiting on
condition [0x00007f8d4e09f000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000066b435850> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
"StoreFileSplitter-2" prio=10 tid=0x00007f8dd404f800 nid=0xac1 waiting on
condition [0x00007f8d4e5a4000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000066b435850> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
"StoreFileSplitter-1" prio=10 tid=0x00007f8dd4013000 nid=0xac0 waiting on
condition [0x00007f8d4f3b2000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000066b435850> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
"StoreFileSplitter-0" prio=10 tid=0x00007f8dd4011000 nid=0xabf waiting on
condition [0x00007f8d4f0af000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000066b435850> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
"DataStreamer for file
/hbase/TestTable/a1df308cf65d4ce008ff42ffabe2667d/splits/8d293d6fcbf8dfd8400a03716822220e/info/a2cc6d762d914bb289fd6d5e558eabad.a1df308cf65d4ce008ff42ffabe2667d"
daemon prio=10 tid=0x00007f8dc400e000 nid=0x95a in Object.wait()
[0x00007f8d4dd9c000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at
org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:440)
- locked <0x00000006598abe20> (a java.util.LinkedList)
"DataStreamer for file
/hbase/TestTable/a1df308cf65d4ce008ff42ffabe2667d/splits/11addefa90702a025b98531ccf79da4f/info/22ed2a4bc9b140a398906fd25d91cc7b.a1df308cf65d4ce008ff42ffabe2667d"
daemon prio=10 tid=0x00007f8e0800b800 nid=0x94f in Object.wait()
[0x00007f8d4da99000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at
org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:440)
- locked <0x00000006be1c7698> (a java.util.LinkedList)
"DataStreamer for file
/hbase/TestTable/a1df308cf65d4ce008ff42ffabe2667d/splits/11addefa90702a025b98531ccf79da4f/info/e54222e2a1dc4688a724157dacdc415d.a1df308cf65d4ce008ff42ffabe2667d"
daemon prio=10 tid=0x00007f8dd8006800 nid=0x94e in Object.wait()
[0x00007f8d4dc9b000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at
org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:440)
- locked <0x000000062e38d4b8> (a java.util.LinkedList)
"DataStreamer for file
/hbase/TestTable/a1df308cf65d4ce008ff42ffabe2667d/splits/11addefa90702a025b98531ccf79da4f/info/869f897364b84903b939686d6aff422a.a1df308cf65d4ce008ff42ffabe2667d"
daemon prio=10 tid=0x00007f8dcc038000 nid=0x94d in Object.wait()
[0x00007f8d4ecab000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at
org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:440)
- locked <0x00000006be1c7718> (a java.util.LinkedList)
"RS_OPEN_REGION-hadoop1,60020,1365731099431-2" prio=10
tid=0x00007f8d78004000 nid=0x6ef0 waiting on condition [0x00007f8d4f4b3000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000061b3d3c28> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
"RS_OPEN_REGION-hadoop1,60020,1365731099431-1" prio=10
tid=0x00007f8d78003000 nid=0x6eef waiting on condition [0x00007f8d4f1b0000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000061b3d3c28> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
"RS_OPEN_REGION-hadoop1,60020,1365731099431-0" prio=10
tid=0x00007f8d78002000 nid=0x6eee waiting on condition [0x00007f8f5c1a1000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000061b3d3c28> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
"RS_OPEN_ROOT-hadoop1,60020,1365731099431-0" prio=10 tid=0x00007f8d64002800
nid=0x6ede waiting on condition [0x00007f8d4f2b1000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000061ae22178> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
"Timer thread for monitoring jvm" daemon prio=10 tid=0x00007f8eec2c5800
nid=0x6deb in Object.wait() [0x00007f8f57707000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x000000061ae4a330> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"Timer thread for monitoring hbase" daemon prio=10 tid=0x00007f8eec2e2000
nid=0x6dea in Object.wait() [0x00007f8f57808000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x000000061ae2c468> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"LeaseRenewer:hadoop@mycluster" daemon prio=10 tid=0x00007f8eec22e800
nid=0x6de8 waiting on condition [0x00007f8f57a0a000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:432)
at
org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:70)
at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:297)
at java.lang.Thread.run(Thread.java:722)
"DestroyJavaVM" prio=10 tid=0x00007f8f7000e000 nid=0x6cbf waiting on
condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"HftpFileSystem-DelegationTokenRenewer" daemon prio=10
tid=0x00007f8f70fad800 nid=0x6dd9 waiting on condition [0x00007f8f5c4a4000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000061ae2d060> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at java.util.concurrent.DelayQueue.take(DelayQueue.java:209)
at
org.apache.hadoop.fs.DelegationTokenRenewer.run(DelegationTokenRenewer.java:152)
"regionserver60020" prio=10 tid=0x00007f8f70f77800 nid=0x6dd6 waiting on
condition [0x00007f8f5c9ab000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000061b38ad38> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.ThreadPoolExecutor.awaitTermination(ThreadPoolExecutor.java:1433)
at
org.apache.hadoop.hbase.regionserver.CompactSplitThread.waitFor(CompactSplitThread.java:240)
at
org.apache.hadoop.hbase.regionserver.CompactSplitThread.join(CompactSplitThread.java:249)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.join(HRegionServer.java:1598)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:760)
at java.lang.Thread.run(Thread.java:722)
"Timer thread for monitoring rpc" daemon prio=10 tid=0x00007f8f70ea8800
nid=0x6dcf in Object.wait() [0x00007f8f5ccae000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x000000061ae22208> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"Service Thread" daemon prio=10 tid=0x00007f8f70317000 nid=0x6cdd runnable
[0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread1" daemon prio=10 tid=0x00007f8f70314800 nid=0x6cdc
waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" daemon prio=10 tid=0x00007f8f70312800 nid=0x6cdb
waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Signal Dispatcher" daemon prio=10 tid=0x00007f8f70310000 nid=0x6cda
runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Surrogate Locker Thread (Concurrent GC)" daemon prio=10
tid=0x00007f8f7030e000 nid=0x6cd9 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" daemon prio=10 tid=0x00007f8f70268000 nid=0x6cd7 in
Object.wait() [0x00007f8f5f2d8000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135)
- locked <0x000000061ae222c8> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:177)
"Reference Handler" daemon prio=10 tid=0x00007f8f70266000 nid=0x6cd6 in
Object.wait() [0x00007f8f5f3d9000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:503)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133)
- locked <0x000000061ae49c28> (a java.lang.ref.Reference$Lock)
"VM Thread" prio=10 tid=0x00007f8f7025e800 nid=0x6cd5 runnable
"Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x00007f8f70014800
nid=0x6cc0 runnable
"Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x00007f8f70016000
nid=0x6cc1 runnable
"Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x00007f8f70018000
nid=0x6cc2 runnable
"Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x00007f8f7001a000
nid=0x6cc3 runnable
"Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x00007f8f7001b800
nid=0x6cc4 runnable
"Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x00007f8f7001d800
nid=0x6cc5 runnable
"Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x00007f8f7001f800
nid=0x6cc6 runnable
"Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x00007f8f70021000
nid=0x6cc7 runnable
"Gang worker#8 (Parallel GC Threads)" prio=10 tid=0x00007f8f70023000
nid=0x6cc8 runnable
"Gang worker#9 (Parallel GC Threads)" prio=10 tid=0x00007f8f70025000
nid=0x6cc9 runnable
"Gang worker#10 (Parallel GC Threads)" prio=10 tid=0x00007f8f70027000
nid=0x6cca runnable
"Gang worker#11 (Parallel GC Threads)" prio=10 tid=0x00007f8f70028800
nid=0x6ccb runnable
"Gang worker#12 (Parallel GC Threads)" prio=10 tid=0x00007f8f7002a800
nid=0x6ccc runnable
"Gang worker#13 (Parallel GC Threads)" prio=10 tid=0x00007f8f7002c800
nid=0x6ccd runnable
"Gang worker#14 (Parallel GC Threads)" prio=10 tid=0x00007f8f7002e000
nid=0x6cce runnable
"Gang worker#15 (Parallel GC Threads)" prio=10 tid=0x00007f8f70030000
nid=0x6ccf runnable
"Concurrent Mark-Sweep GC Thread" prio=10 tid=0x00007f8f701e9800 nid=0x6cd4
runnable
"Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x00007f8f701e1800
nid=0x6cd0 runnable
"Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x00007f8f701e3800
nid=0x6cd1 runnable
"Gang worker#2 (Parallel CMS Threads)" prio=10 tid=0x00007f8f701e5000
nid=0x6cd2 runnable
"Gang worker#3 (Parallel CMS Threads)" prio=10 tid=0x00007f8f701e7000
nid=0x6cd3 runnable
"VM Periodic Task Thread" prio=10 tid=0x00007f8f7032a800 nid=0x6cde waiting
on condition
JNI global references: 160
On Wed, Apr 17, 2013 at 4:26 PM, hua beatls <[email protected]> wrote:
> any good tool for thread dump? can you recommand?
>
> Thanks!
>
> beatls
>
>
> On Wed, Apr 17, 2013 at 4:06 PM, ramkrishna vasudevan <
> [email protected]> wrote:
>
>> Can you attach a thread dump for this ? Which version of HBase are you
>> using.
>>
>> Logs also if attached would be fine.
>>
>> Regards
>> Ram
>>
>>
>> On Wed, Apr 17, 2013 at 1:07 PM, hua beatls <[email protected]> wrote:
>>
>> > HI,
>> > from web ui I find one of my 5 regionserver missing, and check the
>> log
>> > find:
>> >
>> >
>> *****************************************************************************************
>> > [hadoop@hadoop1 logs]$ tail -f hbase-hadoop-regionserver-hadoop1.log
>> > 2013-04-17 15:21:24,789 DEBUG
>> > org.apache.hadoop.hbase.regionserver.CompactSplitThread: Waiting for
>> Split
>> > Thread to finish...
>> > 2013-04-17 15:22:24,789 DEBUG
>> > org.apache.hadoop.hbase.regionserver.CompactSplitThread: Waiting for
>> Split
>> > Thread to finish...
>> >
>> >
>> *********************************************************************************************
>> > but the regionserver process is still alive from jps output.
>> >
>> > any suggestion?
>> >
>> > Thanks!
>> >
>> > beatls
>> >
>>
>
>
