[
https://issues.apache.org/jira/browse/TEZ-1827?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14240610#comment-14240610
]
Ashutosh Chauhan commented on TEZ-1827:
---------------------------------------
jstack appmaster
{code}
Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.55-b03 mixed mode):
"Attach Listener" daemon prio=5 tid=0x00007fb1bd807800 nid=0xed0b waiting on
condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"IPC Client (448170679) connection to localhost/127.0.0.1:61475 from ashutosh"
daemon prio=5 tid=0x00007fb1bc56b800 nid=0x5807 in Object.wait()
[0x000000011ee6b000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007f646feb8> (a
org.apache.hadoop.ipc.Client$Connection)
at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:904)
- locked <0x00000007f646feb8> (a
org.apache.hadoop.ipc.Client$Connection)
at org.apache.hadoop.ipc.Client$Connection.run(Client.java:949)
"ResponseProcessor for block
BP-1442501582-10.10.10.15-1418176179742:blk_1073742793_1969" daemon prio=5
tid=0x00007fb1bb1ff800 nid=0xde03 runnable [0x000000011f47d000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000007f76f8f30> (a sun.nio.ch.Util$2)
- locked <0x00000007f76f8f20> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007f76f8cf0> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at
org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
at
org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
at
org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
at
org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
at
org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118)
at java.io.FilterInputStream.read(FilterInputStream.java:83)
at java.io.FilterInputStream.read(FilterInputStream.java:83)
at
org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:1998)
at
org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:176)
at
org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer$ResponseProcessor.run(DFSOutputStream.java:798)
"LeaseRenewer:ashutosh@localhost:61475" daemon prio=5 tid=0x00007fb1bc041000
nid=0x3d07 waiting on condition [0x000000011f37a000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:438)
at org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:71)
at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:298)
at java.lang.Thread.run(Thread.java:745)
"DataStreamer for file
ashutosh/_tez_session_dir/616ff32f-acb9-40d7-a3c5-7135a92564b9/.tez/application_1418176183755_0039/recovery/1/summary
block BP-1442501582-10.10.10.15-1418176179742:blk_1073742793_1969" daemon
prio=5 tid=0x00007fb1bac7a800 nid=0x3b07 in Object.wait() [0x0000000118396000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007f703f688> (a java.util.LinkedList)
at
org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:503)
- locked <0x00000007f703f688> (a java.util.LinkedList)
"IPC Parameter Sending Thread #1" daemon prio=5 tid=0x00007fb1bd0b6000
nid=0xdb03 waiting on condition [0x000000011f277000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fa0078> (a
java.util.concurrent.SynchronousQueue$TransferStack)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"IPC Parameter Sending Thread #2" daemon prio=5 tid=0x00007fb1bac97800
nid=0xd903 waiting on condition [0x000000011f174000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fa0078> (a
java.util.concurrent.SynchronousQueue$TransferStack)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"ContainerLauncher #2" daemon prio=5 tid=0x00007fb1bb720800 nid=0xd103 waiting
on condition [0x000000011ed68000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fa0370> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"ContainerLauncher #1" daemon prio=5 tid=0x00007fb1bc5c7000 nid=0xcf03 waiting
on condition [0x000000011ec65000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fa0370> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"ContainerLauncher #0" daemon prio=5 tid=0x00007fb1bc4cd000 nid=0xcd03 waiting
on condition [0x000000011eb62000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fa0370> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"DestroyJavaVM" prio=5 tid=0x00007fb1bc0c5000 nid=0x1903 waiting on condition
[0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Timer-0" daemon prio=5 tid=0x00007fb1bab3f000 nid=0xcb03 in Object.wait()
[0x000000011ea5f000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007f6fb81a0> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x00000007f6fb81a0> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"Timer for 'DAGAppMaster' metrics system" daemon prio=5 tid=0x00007fb1bc0c4800
nid=0x6f07 in Object.wait() [0x000000011bcd8000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007f6fb8260> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x00000007f6fb8260> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"TaskSchedulerEventHandlerThread" prio=5 tid=0x00007fb1bac94800 nid=0xc903
waiting on condition [0x000000011e95c000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8320> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
org.apache.tez.dag.app.rm.TaskSchedulerEventHandler$1.run(TaskSchedulerEventHandler.java:350)
"DelayedContainerManager" prio=5 tid=0x00007fb1bac85800 nid=0xc703 in
Object.wait() [0x000000011e859000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007f6fa81d8> (a
org.apache.tez.dag.app.rm.YarnTaskSchedulerService$DelayedContainerManager)
at
org.apache.tez.dag.app.rm.YarnTaskSchedulerService$DelayedContainerManager.run(YarnTaskSchedulerService.java:1784)
- locked <0x00000007f6fa81d8> (a
org.apache.tez.dag.app.rm.YarnTaskSchedulerService$DelayedContainerManager)
"TaskSchedulerAppCaller #0" daemon prio=5 tid=0x00007fb1bac85000 nid=0xc503
waiting on condition [0x000000011e756000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fc0120> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"AMRM Heartbeater thread" prio=5 tid=0x00007fb1bac80000 nid=0xc303 waiting on
condition [0x000000011e653000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.hadoop.yarn.client.api.async.impl.AMRMClientAsyncImpl$HeartbeatThread.run(AMRMClientAsyncImpl.java:249)
"IPC Client (448170679) connection to /10.10.10.15:61494 from ashutosh" daemon
prio=5 tid=0x00007fb1bb9fe800 nid=0xc103 in Object.wait() [0x000000011e550000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007f6fc0220> (a
org.apache.hadoop.ipc.Client$Connection)
at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:904)
- locked <0x00000007f6fc0220> (a
org.apache.hadoop.ipc.Client$Connection)
at org.apache.hadoop.ipc.Client$Connection.run(Client.java:949)
"AMRM Callback Handler Thread" daemon prio=5 tid=0x00007fb1bb9fc800 nid=0x6907
waiting on condition [0x000000011b9cf000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fc0358> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
org.apache.hadoop.yarn.client.api.async.impl.AMRMClientAsyncImpl$CallbackHandlerThread.run(AMRMClientAsyncImpl.java:275)
"IPC Server handler 29 on 54326" daemon prio=5 tid=0x00007fb1be010800
nid=0xbf03 waiting on condition [0x000000011e44d000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 28 on 54326" daemon prio=5 tid=0x00007fb1be010000
nid=0xbd03 waiting on condition [0x000000011e34a000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 27 on 54326" daemon prio=5 tid=0x00007fb1be04f000
nid=0xbb03 waiting on condition [0x000000011e247000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 26 on 54326" daemon prio=5 tid=0x00007fb1bb9fc000
nid=0xb903 waiting on condition [0x000000011e144000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 25 on 54326" daemon prio=5 tid=0x00007fb1bb9f8800
nid=0xb703 waiting on condition [0x000000011e041000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 24 on 54326" daemon prio=5 tid=0x00007fb1bb9f8000
nid=0xb503 waiting on condition [0x000000011df3e000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 23 on 54326" daemon prio=5 tid=0x00007fb1bba85800
nid=0xb303 waiting on condition [0x000000011de3b000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 22 on 54326" daemon prio=5 tid=0x00007fb1bba85000
nid=0xb103 waiting on condition [0x000000011dd38000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 21 on 54326" daemon prio=5 tid=0x00007fb1bba78800
nid=0xaf03 waiting on condition [0x000000011dc35000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 20 on 54326" daemon prio=5 tid=0x00007fb1bba78000
nid=0xad03 waiting on condition [0x000000011db32000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 19 on 54326" daemon prio=5 tid=0x00007fb1bb80e800
nid=0xab03 waiting on condition [0x000000011da2f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 18 on 54326" daemon prio=5 tid=0x00007fb1bb80d800
nid=0xa903 waiting on condition [0x000000011d92c000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 17 on 54326" daemon prio=5 tid=0x00007fb1bb8c9000
nid=0xa703 waiting on condition [0x000000011d829000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 16 on 54326" daemon prio=5 tid=0x00007fb1bb8c8000
nid=0xa503 waiting on condition [0x000000011d726000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 15 on 54326" daemon prio=5 tid=0x00007fb1bb9d3000
nid=0xa303 waiting on condition [0x000000011d623000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 14 on 54326" daemon prio=5 tid=0x00007fb1bb9e5000
nid=0xa103 waiting on condition [0x000000011d520000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 13 on 54326" daemon prio=5 tid=0x00007fb1bb9f0800
nid=0x9f03 waiting on condition [0x000000011d41d000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 12 on 54326" daemon prio=5 tid=0x00007fb1bd077800
nid=0x9d03 waiting on condition [0x000000011d31a000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 11 on 54326" daemon prio=5 tid=0x00007fb1be855800
nid=0x9b03 waiting on condition [0x000000011d217000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 10 on 54326" daemon prio=5 tid=0x00007fb1be855000
nid=0x6407 waiting on condition [0x0000000119fdd000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 9 on 54326" daemon prio=5 tid=0x00007fb1be854000 nid=0x9903
waiting on condition [0x000000011d114000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 8 on 54326" daemon prio=5 tid=0x00007fb1be85b000 nid=0x9703
waiting on condition [0x000000011d011000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 7 on 54326" daemon prio=5 tid=0x00007fb1bab40000 nid=0x9503
waiting on condition [0x000000011cf0e000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 6 on 54326" daemon prio=5 tid=0x00007fb1bb8c5800 nid=0x9303
waiting on condition [0x000000011ce0b000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 5 on 54326" daemon prio=5 tid=0x00007fb1bb8c4800 nid=0x9103
waiting on condition [0x000000011cd08000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 4 on 54326" daemon prio=5 tid=0x00007fb1be859800 nid=0x8f03
waiting on condition [0x000000011cc05000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 3 on 54326" daemon prio=5 tid=0x00007fb1bb80b000 nid=0x8d03
waiting on condition [0x000000011cb02000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 2 on 54326" daemon prio=5 tid=0x00007fb1bb80a800 nid=0x8b03
waiting on condition [0x000000011c9ff000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 1 on 54326" daemon prio=5 tid=0x00007fb1bd077000 nid=0x8903
waiting on condition [0x000000011c8fc000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 0 on 54326" daemon prio=5 tid=0x00007fb1bb8c1800 nid=0x8703
waiting on condition [0x000000011c7f9000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fb8418> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server listener on 54326" daemon prio=5 tid=0x00007fb1be04b800 nid=0x8503
runnable [0x000000011c6f6000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000007f7077cd8> (a sun.nio.ch.Util$2)
- locked <0x00000007f7077ce8> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007f7077c88> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
at org.apache.hadoop.ipc.Server$Listener.run(Server.java:668)
"IPC Server Responder" daemon prio=5 tid=0x00007fb1baaa8000 nid=0x8303 runnable
[0x000000011c5f3000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000007f7086688> (a sun.nio.ch.Util$2)
- locked <0x00000007f7086698> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007f7086638> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:835)
at org.apache.hadoop.ipc.Server$Responder.run(Server.java:818)
"IPC Server handler 0 on 54327" daemon prio=5 tid=0x00007fb1bb8ba000 nid=0x8103
waiting on condition [0x000000011c4f0000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f70cb0d0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server listener on 54327" daemon prio=5 tid=0x00007fb1bb8b9800 nid=0x7f03
runnable [0x000000011c3ed000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000007f7086a80> (a sun.nio.ch.Util$2)
- locked <0x00000007f7086a90> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007f7086a30> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
at org.apache.hadoop.ipc.Server$Listener.run(Server.java:668)
"IPC Server Responder" daemon prio=5 tid=0x00007fb1bb9e6000 nid=0x7307 runnable
[0x000000011bede000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000007f6f99760> (a sun.nio.ch.Util$2)
- locked <0x00000007f6f99770> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007f6f99710> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:835)
at org.apache.hadoop.ipc.Server$Responder.run(Server.java:818)
"RecoveryEventHandlingThread" prio=5 tid=0x00007fb1bd909800 nid=0x7d03 waiting
on condition [0x000000011c2ea000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6f99ac0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
org.apache.tez.dag.history.recovery.RecoveryService$1.run(RecoveryService.java:149)
at java.lang.Thread.run(Thread.java:745)
"HistoryEventHandlingThread" prio=5 tid=0x00007fb1bb234800 nid=0x7b03 waiting
on condition [0x000000011bddb000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f70d2138> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
org.apache.tez.dag.history.logging.impl.SimpleHistoryLoggingService$1.run(SimpleHistoryLoggingService.java:103)
at java.lang.Thread.run(Thread.java:745)
"IPC Server idle connection scanner for port 54326" daemon prio=5
tid=0x00007fb1bac7f000 nid=0x6d07 in Object.wait() [0x000000011bbd5000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007f70cdae0> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x00000007f70cdae0> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"IPC Server idle connection scanner for port 54327" daemon prio=5
tid=0x00007fb1bc3dd000 nid=0x6b0b in Object.wait() [0x000000011bad2000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007f6f99d48> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x00000007f6f99d48> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"process reaper" daemon prio=5 tid=0x00007fb1bc109000 nid=0x6707 waiting on
condition [0x0000000118973000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6f80dc8> (a
java.util.concurrent.SynchronousQueue$TransferStack)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"Socket Reader #1 for port 54326" prio=5 tid=0x00007fb1bab36800 nid=0x6507
runnable [0x000000011a1e3000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000007f7114b78> (a sun.nio.ch.Util$2)
- locked <0x00000007f7114b88> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007f7077f40> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
at
org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:616)
- locked <0x00000007f7114ad8> (a
org.apache.hadoop.ipc.Server$Listener$Reader)
at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:595)
"Socket Reader #1 for port 54327" prio=5 tid=0x00007fb1bb9d2000 nid=0x7107
runnable [0x000000011a0e0000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000007f7116208> (a sun.nio.ch.Util$2)
- locked <0x00000007f7116218> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007f7086cd8> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
at
org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:616)
- locked <0x00000007f7116168> (a
org.apache.hadoop.ipc.Server$Listener$Reader)
at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:595)
"ContainerLauncher Event Handler" prio=5 tid=0x00007fb1bb988000 nid=0x6107
waiting on condition [0x0000000119eda000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f71176f8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
org.apache.tez.dag.app.launcher.ContainerLauncherImpl$1.run(ContainerLauncherImpl.java:279)
"TaskHeartbeatHandler PingChecker" prio=5 tid=0x00007fb1bb5bb800 nid=0x7903
waiting on condition [0x000000011c1e7000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.tez.dag.app.HeartbeatHandlerBase$PingChecker.run(HeartbeatHandlerBase.java:159)
at java.lang.Thread.run(Thread.java:745)
"ContainerHeartbeatHandler PingChecker" prio=5 tid=0x00007fb1bab39800
nid=0x7703 waiting on condition [0x000000011c0e4000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.tez.dag.app.HeartbeatHandlerBase$PingChecker.run(HeartbeatHandlerBase.java:159)
at java.lang.Thread.run(Thread.java:745)
"AsyncDispatcher event handler" prio=5 tid=0x00007fb1baac9800 nid=0x7503
waiting on condition [0x000000011bfe1000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f7119628> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:98)
at java.lang.Thread.run(Thread.java:745)
"IPC Parameter Sending Thread #0" daemon prio=5 tid=0x00007fb1babf2800
nid=0x5f03 waiting on condition [0x0000000119dd7000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f6fa0078> (a
java.util.concurrent.SynchronousQueue$TransferStack)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"Service Thread" daemon prio=5 tid=0x00007fb1bb809000 nid=0x5303 runnable
[0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread1" daemon prio=5 tid=0x00007fb1bc05e000 nid=0x5103 waiting on
condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" daemon prio=5 tid=0x00007fb1bb808000 nid=0x4f03 waiting on
condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Signal Dispatcher" daemon prio=5 tid=0x00007fb1bc056800 nid=0x4d03 runnable
[0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" daemon prio=5 tid=0x00007fb1bb004000 nid=0x3903 in Object.wait()
[0x000000011825a000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007f6f8bc90> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135)
- locked <0x00000007f6f8bc90> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:189)
"Reference Handler" daemon prio=5 tid=0x00007fb1ba830000 nid=0x3703 in
Object.wait() [0x0000000118157000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007f6f8b6f0> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:503)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133)
- locked <0x00000007f6f8b6f0> (a java.lang.ref.Reference$Lock)
"VM Thread" prio=5 tid=0x00007fb1ba82f800 nid=0x3503 runnable
"GC task thread#0 (ParallelGC)" prio=5 tid=0x00007fb1ba82e000 nid=0x2503
runnable
"GC task thread#1 (ParallelGC)" prio=5 tid=0x00007fb1ba82f000 nid=0x2703
runnable
"GC task thread#2 (ParallelGC)" prio=5 tid=0x00007fb1bb800000 nid=0x2903
runnable
"GC task thread#3 (ParallelGC)" prio=5 tid=0x00007fb1bc000000 nid=0x2b03
runnable
"GC task thread#4 (ParallelGC)" prio=5 tid=0x00007fb1bc001000 nid=0x2d03
runnable
"GC task thread#5 (ParallelGC)" prio=5 tid=0x00007fb1bc001800 nid=0x2f03
runnable
"GC task thread#6 (ParallelGC)" prio=5 tid=0x00007fb1bc002000 nid=0x3103
runnable
"GC task thread#7 (ParallelGC)" prio=5 tid=0x00007fb1bc002800 nid=0x3303
runnable
"VM Periodic Task Thread" prio=5 tid=0x00007fb1bb809800 nid=0x5503 waiting on
condition
JNI global references: 284
{code}
{code}
$ jstack tezChild
"Attach Listener" daemon prio=5 tid=0x00007fe4aaa55800 nid=0x681b waiting on
condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"org.apache.hadoop.hdfs.PeerCache@5b8baa5f" daemon prio=5
tid=0x00007fe4aca25800 nid=0x6c03 waiting on condition [0x000000011b76c000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:244)
at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:41)
at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:119)
at java.lang.Thread.run(Thread.java:745)
"TaskHeartbeatThread" daemon prio=5 tid=0x00007fe4abb01000 nid=0x6603 waiting
on condition [0x000000011bb6a000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd6953c0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"IPC Parameter Sending Thread #0" daemon prio=5 tid=0x00007fe4ac2db000
nid=0x6403 waiting on condition [0x000000011b669000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd49a5e8> (a
java.util.concurrent.SynchronousQueue$TransferStack)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"IPC Client (1216919532) connection to /10.10.10.15:54326 from
application_1418176183755_0039" daemon prio=5 tid=0x00007fe4aaa06000 nid=0x6203
in Object.wait() [0x000000011b566000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007cd491540> (a
org.apache.hadoop.ipc.Client$Connection)
at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:904)
- locked <0x00000007cd491540> (a
org.apache.hadoop.ipc.Client$Connection)
at org.apache.hadoop.ipc.Client$Connection.run(Client.java:949)
"TezChild" daemon prio=5 tid=0x00007fe4ab966800 nid=0x6003 waiting on condition
[0x000000011b463000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at java.lang.Thread.sleep(Thread.java:340)
at java.util.concurrent.TimeUnit.sleep(TimeUnit.java:360)
at
org.apache.tez.runtime.task.ContainerReporter.call(ContainerReporter.java:62)
at
org.apache.tez.runtime.task.ContainerReporter.call(ContainerReporter.java:34)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"Timer for 'TezTask' metrics system" daemon prio=5 tid=0x00007fe4aa93a800
nid=0x5e07 in Object.wait() [0x000000011b360000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007cd48cb48> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x00000007cd48cb48> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"Service Thread" daemon prio=5 tid=0x00007fe4aa874000 nid=0x5303 runnable
[0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread1" daemon prio=5 tid=0x00007fe4aa81f000 nid=0x5103 waiting on
condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" daemon prio=5 tid=0x00007fe4aa872800 nid=0x4f03 waiting on
condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Signal Dispatcher" daemon prio=5 tid=0x00007fe4aa858800 nid=0x4d03 runnable
[0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" daemon prio=5 tid=0x00007fe4ac008000 nid=0x3903 in Object.wait()
[0x000000011846c000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007cd48a5d8> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135)
- locked <0x00000007cd48a5d8> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:189)
"Reference Handler" daemon prio=5 tid=0x00007fe4ac005000 nid=0x3703 in
Object.wait() [0x0000000118369000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007cd48cda8> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:503)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133)
- locked <0x00000007cd48cda8> (a java.lang.ref.Reference$Lock)
"main" prio=5 tid=0x00007fe4aa80f000 nid=0x1903 waiting on condition
[0x000000010ed9e000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007f80c4cd8> (a
com.google.common.util.concurrent.ListenableFutureTask)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:425)
at java.util.concurrent.FutureTask.get(FutureTask.java:187)
at org.apache.tez.runtime.task.TezChild.run(TezChild.java:184)
at org.apache.tez.runtime.task.TezChild.main(TezChild.java:433)
"VM Thread" prio=5 tid=0x00007fe4ac004800 nid=0x3503 runnable
"GC task thread#0 (ParallelGC)" prio=5 tid=0x00007fe4aa823000 nid=0x2503
runnable
"GC task thread#1 (ParallelGC)" prio=5 tid=0x00007fe4aa823800 nid=0x2703
runnable
"GC task thread#2 (ParallelGC)" prio=5 tid=0x00007fe4aa824000 nid=0x2903
runnable
"GC task thread#3 (ParallelGC)" prio=5 tid=0x00007fe4aa824800 nid=0x2b03
runnable
"GC task thread#4 (ParallelGC)" prio=5 tid=0x00007fe4aa825800 nid=0x2d03
runnable
"GC task thread#5 (ParallelGC)" prio=5 tid=0x00007fe4aa826000 nid=0x2f03
runnable
"GC task thread#6 (ParallelGC)" prio=5 tid=0x00007fe4aa826800 nid=0x3103
runnable
"GC task thread#7 (ParallelGC)" prio=5 tid=0x00007fe4aa827000 nid=0x3303
runnable
"VM Periodic Task Thread" prio=5 tid=0x00007fe4aa87d000 nid=0x5503 waiting on
condition
JNI global references: 266
{code}
> MiniTezCluster takes 10 minutes to shut down
> --------------------------------------------
>
> Key: TEZ-1827
> URL: https://issues.apache.org/jira/browse/TEZ-1827
> Project: Apache Tez
> Issue Type: Bug
> Affects Versions: 0.5.2
> Reporter: Ashutosh Chauhan
> Priority: Minor
>
> Always reproducible
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
