[
https://issues.apache.org/jira/browse/TEZ-1827?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14240614#comment-14240614
]
Ashutosh Chauhan commented on TEZ-1827:
---------------------------------------
>From a different run:
{code}
$ jstack appmaster
Picked up JAVA_TOOL_OPTIONS: -Djava.awt.headless=true
2014-12-09 20:01:25
Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.55-b03 mixed mode):
"Attach Listener" daemon prio=5 tid=0x00007fd88e104800 nid=0xe17b waiting on
condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"AMShutdownThread" daemon prio=5 tid=0x00007fd88b18c800 nid=0xc20f waiting for
monitor entry [0x000000011713a000]
java.lang.Thread.State: BLOCKED (on object monitor)
at
org.apache.tez.dag.app.launcher.ContainerLauncherImpl$Container.kill(ContainerLauncherImpl.java:196)
- waiting to lock <0x00000007fe19b408> (a
org.apache.tez.dag.app.launcher.ContainerLauncherImpl$Container)
at
org.apache.tez.dag.app.launcher.ContainerLauncherImpl.shutdownAllContainers(ContainerLauncherImpl.java:325)
at
org.apache.tez.dag.app.launcher.ContainerLauncherImpl.serviceStop(ContainerLauncherImpl.java:337)
at
org.apache.hadoop.service.AbstractService.stop(AbstractService.java:221)
- locked <0x00000007cd710680> (a java.lang.Object)
at
org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:52)
at
org.apache.hadoop.service.ServiceOperations.stopQuietly(ServiceOperations.java:80)
at
org.apache.tez.dag.app.DAGAppMaster.stopServices(DAGAppMaster.java:1504)
at
org.apache.tez.dag.app.DAGAppMaster.serviceStop(DAGAppMaster.java:1643)
- locked <0x00000007cd39d468> (a org.apache.tez.dag.app.DAGAppMaster)
at
org.apache.hadoop.service.AbstractService.stop(AbstractService.java:221)
- locked <0x00000007cd39d748> (a java.lang.Object)
at
org.apache.tez.dag.app.DAGAppMaster$DAGAppMasterShutdownHandler$AMShutdownRunnable.run(DAGAppMaster.java:698)
at java.lang.Thread.run(Thread.java:745)
"ContainerLauncher #9" daemon prio=5 tid=0x00007fd88a903800 nid=0xf303 waiting
on condition [0x000000011d83b000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd710850> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"ContainerLauncher #8" daemon prio=5 tid=0x00007fd889887800 nid=0xed17 waiting
on condition [0x000000011d062000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd710850> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"ContainerLauncher #7" daemon prio=5 tid=0x00007fd88988d000 nid=0x591b waiting
on condition [0x0000000116f9c000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd710850> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"ContainerLauncher #6" daemon prio=5 tid=0x00007fd88db84800 nid=0xf013 waiting
on condition [0x000000011c747000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd710850> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"ContainerLauncher #5" daemon prio=5 tid=0x00007fd88db83800 nid=0xea1b waiting
on condition [0x000000011c644000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd710850> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"ContainerLauncher #4" daemon prio=5 tid=0x00007fd88d89c800 nid=0xd413 waiting
on condition [0x000000011cc56000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd710850> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"ContainerLauncher #3" daemon prio=5 tid=0x00007fd88d898000 nid=0xe407 runnable
[0x000000011c94c000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000007cdd32298> (a sun.nio.ch.Util$2)
- locked <0x00000007cdd322a8> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007cdd32248> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at
org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:203)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:529)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:493)
at
org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:606)
- locked <0x00000007fe1e4630> (a
org.apache.hadoop.ipc.Client$Connection)
at
org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:700)
- locked <0x00000007fe1e4630> (a
org.apache.hadoop.ipc.Client$Connection)
at org.apache.hadoop.ipc.Client$Connection.access$2800(Client.java:367)
at org.apache.hadoop.ipc.Client.getConnection(Client.java:1463)
at org.apache.hadoop.ipc.Client.call(Client.java:1382)
at org.apache.hadoop.ipc.Client.call(Client.java:1364)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206)
at com.sun.proxy.$Proxy18.stopContainers(Unknown Source)
at
org.apache.hadoop.yarn.api.impl.pb.client.ContainerManagementProtocolPBClientImpl.stopContainers(ContainerManagementProtocolPBClientImpl.java:110)
at
org.apache.tez.dag.app.launcher.ContainerLauncherImpl$Container.kill(ContainerLauncherImpl.java:215)
- locked <0x00000007cdc4ebf8> (a
org.apache.tez.dag.app.launcher.ContainerLauncherImpl$Container)
at
org.apache.tez.dag.app.launcher.ContainerLauncherImpl$EventProcessor.run(ContainerLauncherImpl.java:384)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"ContainerLauncher #2" daemon prio=5 tid=0x00007fd88bb51800 nid=0xd203 waiting
for monitor entry [0x000000011c541000]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.apache.hadoop.ipc.Client$Connection.addCall(Client.java:455)
- waiting to lock <0x00000007fe19bde8> (a
org.apache.hadoop.ipc.Client$Connection)
at org.apache.hadoop.ipc.Client$Connection.access$2700(Client.java:367)
at org.apache.hadoop.ipc.Client.getConnection(Client.java:1457)
at org.apache.hadoop.ipc.Client.call(Client.java:1382)
at org.apache.hadoop.ipc.Client.call(Client.java:1364)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206)
at com.sun.proxy.$Proxy18.stopContainers(Unknown Source)
at
org.apache.hadoop.yarn.api.impl.pb.client.ContainerManagementProtocolPBClientImpl.stopContainers(ContainerManagementProtocolPBClientImpl.java:110)
at
org.apache.tez.dag.app.launcher.ContainerLauncherImpl$Container.kill(ContainerLauncherImpl.java:215)
- locked <0x00000007fe19b408> (a
org.apache.tez.dag.app.launcher.ContainerLauncherImpl$Container)
at
org.apache.tez.dag.app.launcher.ContainerLauncherImpl$EventProcessor.run(ContainerLauncherImpl.java:384)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"ContainerLauncher #1" daemon prio=5 tid=0x00007fd88d812800 nid=0xd003 runnable
[0x000000011c43d000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000007fe19c5b0> (a sun.nio.ch.Util$2)
- locked <0x00000007fe19c5a0> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007fe19c3a0> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at
org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:203)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:529)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:493)
at
org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:606)
- locked <0x00000007fe19bde8> (a
org.apache.hadoop.ipc.Client$Connection)
at
org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:700)
- locked <0x00000007fe19bde8> (a
org.apache.hadoop.ipc.Client$Connection)
at org.apache.hadoop.ipc.Client$Connection.access$2800(Client.java:367)
at org.apache.hadoop.ipc.Client.getConnection(Client.java:1463)
at org.apache.hadoop.ipc.Client.call(Client.java:1382)
at org.apache.hadoop.ipc.Client.call(Client.java:1364)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206)
at com.sun.proxy.$Proxy18.stopContainers(Unknown Source)
at
org.apache.hadoop.yarn.api.impl.pb.client.ContainerManagementProtocolPBClientImpl.stopContainers(ContainerManagementProtocolPBClientImpl.java:110)
at
org.apache.tez.dag.app.launcher.ContainerLauncherImpl$Container.kill(ContainerLauncherImpl.java:215)
- locked <0x00000007fe19b778> (a
org.apache.tez.dag.app.launcher.ContainerLauncherImpl$Container)
at
org.apache.tez.dag.app.launcher.ContainerLauncherImpl$EventProcessor.run(ContainerLauncherImpl.java:384)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"ContainerLauncher #0" daemon prio=5 tid=0x00007fd889414000 nid=0xce03 waiting
on condition [0x000000011c33b000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd710850> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"DestroyJavaVM" prio=5 tid=0x00007fd88bb60000 nid=0x1903 waiting on condition
[0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Timer for 'DAGAppMaster' metrics system" daemon prio=5 tid=0x00007fd88b122800
nid=0x7007 in Object.wait() [0x00000001194b1000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007cd8aec98> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x00000007cd8aec98> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"TaskSchedulerEventHandlerThread" prio=5 tid=0x00007fd88a867000 nid=0xca03
waiting on condition [0x000000011c135000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd69e970> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
org.apache.tez.dag.app.rm.TaskSchedulerEventHandler$1.run(TaskSchedulerEventHandler.java:350)
"DelayedContainerManager" prio=5 tid=0x00007fd88bb4d000 nid=0xc803 in
Object.wait() [0x000000011c032000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007cd69ea40> (a
org.apache.tez.dag.app.rm.YarnTaskSchedulerService$DelayedContainerManager)
at
org.apache.tez.dag.app.rm.YarnTaskSchedulerService$DelayedContainerManager.run(YarnTaskSchedulerService.java:1784)
- locked <0x00000007cd69ea40> (a
org.apache.tez.dag.app.rm.YarnTaskSchedulerService$DelayedContainerManager)
"TaskSchedulerAppCaller #0" daemon prio=5 tid=0x00007fd88bade000 nid=0xc603
waiting on condition [0x000000011bf2f000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd7342c0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"AMRM Heartbeater thread" prio=5 tid=0x00007fd88bac1000 nid=0xc403 runnable
[0x000000011be2b000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000007fe1e4530> (a sun.nio.ch.Util$2)
- locked <0x00000007fe1e4540> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007fe1e44e0> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at
org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:203)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:529)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:493)
at
org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:606)
- locked <0x00000007fe19cb78> (a
org.apache.hadoop.ipc.Client$Connection)
at
org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:700)
- locked <0x00000007fe19cb78> (a
org.apache.hadoop.ipc.Client$Connection)
at org.apache.hadoop.ipc.Client$Connection.access$2800(Client.java:367)
at org.apache.hadoop.ipc.Client.getConnection(Client.java:1463)
at org.apache.hadoop.ipc.Client.call(Client.java:1382)
at org.apache.hadoop.ipc.Client.call(Client.java:1364)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206)
at com.sun.proxy.$Proxy16.allocate(Unknown Source)
at
org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.allocate(ApplicationMasterProtocolPBClientImpl.java:77)
at sun.reflect.GeneratedMethodAccessor35.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
at com.sun.proxy.$Proxy17.allocate(Unknown Source)
at
org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl.allocate(AMRMClientImpl.java:251)
at
org.apache.hadoop.yarn.client.api.async.impl.AMRMClientAsyncImpl$HeartbeatThread.run(AMRMClientAsyncImpl.java:224)
- locked <0x00000007cd734178> (a java.lang.Object)
"AMRM Callback Handler Thread" daemon prio=5 tid=0x00007fd88b115800 nid=0x6a07
waiting on condition [0x00000001191a8000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd734098> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
org.apache.hadoop.yarn.client.api.async.impl.AMRMClientAsyncImpl$CallbackHandlerThread.run(AMRMClientAsyncImpl.java:275)
"IPC Server handler 29 on 57361" daemon prio=5 tid=0x00007fd88babe000
nid=0xc003 waiting on condition [0x000000011bc26000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 28 on 57361" daemon prio=5 tid=0x00007fd88c9b6000
nid=0xbe03 waiting on condition [0x000000011bb23000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 27 on 57361" daemon prio=5 tid=0x00007fd88d258800
nid=0xbc03 waiting on condition [0x000000011ba20000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 26 on 57361" daemon prio=5 tid=0x00007fd88babd800
nid=0xba03 waiting on condition [0x000000011b91d000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 25 on 57361" daemon prio=5 tid=0x00007fd88babc800
nid=0xb803 waiting on condition [0x000000011b81a000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 24 on 57361" daemon prio=5 tid=0x00007fd88babc000
nid=0xb603 waiting on condition [0x000000011b717000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 23 on 57361" daemon prio=5 tid=0x00007fd88babb000
nid=0xb403 waiting on condition [0x000000011b614000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 22 on 57361" daemon prio=5 tid=0x00007fd88baba800
nid=0xb203 waiting on condition [0x000000011b511000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 21 on 57361" daemon prio=5 tid=0x00007fd88c9b5800
nid=0xb003 waiting on condition [0x000000011b40e000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 20 on 57361" daemon prio=5 tid=0x00007fd88d24f800
nid=0xae03 waiting on condition [0x000000011b30b000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 19 on 57361" daemon prio=5 tid=0x00007fd88d24e800
nid=0xac03 waiting on condition [0x000000011b208000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 18 on 57361" daemon prio=5 tid=0x00007fd88c9b4800
nid=0xaa03 waiting on condition [0x000000011b105000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 17 on 57361" daemon prio=5 tid=0x00007fd88bab8800
nid=0xa803 waiting on condition [0x000000011b002000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 16 on 57361" daemon prio=5 tid=0x00007fd88bab5000
nid=0xa603 waiting on condition [0x000000011aeff000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 15 on 57361" daemon prio=5 tid=0x00007fd88d0c6000
nid=0xa403 waiting on condition [0x000000011adfc000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 14 on 57361" daemon prio=5 tid=0x00007fd88c903000
nid=0xa203 waiting on condition [0x000000011acf9000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 13 on 57361" daemon prio=5 tid=0x00007fd88c902800
nid=0x6507 waiting on condition [0x0000000118e9f000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 12 on 57361" daemon prio=5 tid=0x00007fd88d0c5800
nid=0xa003 waiting on condition [0x000000011abf6000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 11 on 57361" daemon prio=5 tid=0x00007fd88bab9800
nid=0x9e03 waiting on condition [0x000000011aaf3000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 10 on 57361" daemon prio=5 tid=0x00007fd88d19f000
nid=0x9c03 waiting on condition [0x000000011a9f0000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 9 on 57361" daemon prio=5 tid=0x00007fd88bb34000 nid=0x9a03
waiting on condition [0x000000011a8ed000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 8 on 57361" daemon prio=5 tid=0x00007fd88c939800 nid=0x9803
waiting on condition [0x000000011a7ea000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 7 on 57361" daemon prio=5 tid=0x00007fd88c938800 nid=0x9603
waiting on condition [0x000000011a6e7000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 6 on 57361" daemon prio=5 tid=0x00007fd88bab2000 nid=0x9403
waiting on condition [0x000000011a5e4000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 5 on 57361" daemon prio=5 tid=0x00007fd88b115000 nid=0x9203
waiting on condition [0x000000011a4e1000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 4 on 57361" daemon prio=5 tid=0x00007fd88d055800 nid=0x9003
waiting on condition [0x000000011a3de000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 3 on 57361" daemon prio=5 tid=0x00007fd88a972800 nid=0x8e03
waiting on condition [0x000000011a2db000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 2 on 57361" daemon prio=5 tid=0x00007fd88bab1800 nid=0x8c03
waiting on condition [0x000000011a1d8000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 1 on 57361" daemon prio=5 tid=0x00007fd88bab0800 nid=0x8a03
waiting on condition [0x000000011a0d5000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 0 on 57361" daemon prio=5 tid=0x00007fd88baae000 nid=0x8803
waiting on condition [0x0000000119fd2000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd683918> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server handler 0 on 57362" daemon prio=5 tid=0x00007fd88c0c9800 nid=0x8603
waiting on condition [0x0000000119ecf000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd70bd80> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1985)
"IPC Server listener on 57361" daemon prio=5 tid=0x00007fd88b1e3800 nid=0x8403
runnable [0x0000000119dcc000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000007cd6caba8> (a sun.nio.ch.Util$2)
- locked <0x00000007cd6cab98> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007cd6ca7d0> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
at org.apache.hadoop.ipc.Server$Listener.run(Server.java:668)
"IPC Server Responder" daemon prio=5 tid=0x00007fd88b1e2800 nid=0x8203 runnable
[0x0000000119cc9000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000007cd6cb2e8> (a sun.nio.ch.Util$2)
- locked <0x00000007cd6cb2d8> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007cd6cb1b8> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:835)
at org.apache.hadoop.ipc.Server$Responder.run(Server.java:818)
"IPC Server listener on 57362" daemon prio=5 tid=0x00007fd88baad000 nid=0x8003
runnable [0x0000000119bc6000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000007cd72ad60> (a sun.nio.ch.Util$2)
- locked <0x00000007cd72ad50> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007cd72a988> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
at org.apache.hadoop.ipc.Server$Listener.run(Server.java:668)
"IPC Server Responder" daemon prio=5 tid=0x00007fd88bab6000 nid=0x7407 runnable
[0x00000001196b7000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000007cd72b4a0> (a sun.nio.ch.Util$2)
- locked <0x00000007cd72b490> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007cd72b370> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:835)
at org.apache.hadoop.ipc.Server$Responder.run(Server.java:818)
"IPC Server idle connection scanner for port 57362" daemon prio=5
tid=0x00007fd889a40800 nid=0x6807 in Object.wait() [0x00000001193ae000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007cd72a3a0> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x00000007cd72a3a0> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"IPC Server idle connection scanner for port 57361" daemon prio=5
tid=0x00007fd88ba25000 nid=0x6e07 in Object.wait() [0x00000001192ab000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007cd6c0040> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x00000007cd6c0040> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"Socket Reader #1 for port 57362" prio=5 tid=0x00007fd889499000 nid=0x6207
runnable [0x00000001190a5000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000007cd70c188> (a sun.nio.ch.Util$2)
- locked <0x00000007cd70c178> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007cd70c048> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
at
org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:616)
- locked <0x00000007cd72aed8> (a
org.apache.hadoop.ipc.Server$Listener$Reader)
at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:595)
"Socket Reader #1 for port 57361" prio=5 tid=0x00007fd88baf0800 nid=0x7207
runnable [0x0000000118fa2000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x00000007cd6bef08> (a sun.nio.ch.Util$2)
- locked <0x00000007cd6beef8> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007cd6bed80> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
at
org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:616)
- locked <0x00000007cd6cad20> (a
org.apache.hadoop.ipc.Server$Listener$Reader)
at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:595)
"ContainerLauncher Event Handler" prio=5 tid=0x00007fd8892a1800 nid=0x6607
waiting on condition [0x0000000117580000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd710b60> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
org.apache.tez.dag.app.launcher.ContainerLauncherImpl$1.run(ContainerLauncherImpl.java:279)
"TaskHeartbeatHandler PingChecker" prio=5 tid=0x00007fd88c810000 nid=0x7a03
waiting on condition [0x00000001199c0000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.tez.dag.app.HeartbeatHandlerBase$PingChecker.run(HeartbeatHandlerBase.java:159)
at java.lang.Thread.run(Thread.java:745)
"ContainerHeartbeatHandler PingChecker" prio=5 tid=0x00007fd88ba28000
nid=0x7803 waiting on condition [0x00000001198bd000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.tez.dag.app.HeartbeatHandlerBase$PingChecker.run(HeartbeatHandlerBase.java:159)
at java.lang.Thread.run(Thread.java:745)
"AsyncDispatcher event handler" prio=5 tid=0x00007fd889868000 nid=0x7603
waiting on condition [0x00000001197ba000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007cd77a570> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:98)
at java.lang.Thread.run(Thread.java:745)
"Service Thread" daemon prio=5 tid=0x00007fd889069000 nid=0x5303 runnable
[0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread1" daemon prio=5 tid=0x00007fd889068000 nid=0x5103 waiting on
condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" daemon prio=5 tid=0x00007fd889066800 nid=0x4f03 waiting on
condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Signal Dispatcher" daemon prio=5 tid=0x00007fd889065800 nid=0x4d03 runnable
[0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" daemon prio=5 tid=0x00007fd889062000 nid=0x3903 in Object.wait()
[0x000000011599f000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007cd34be68> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135)
- locked <0x00000007cd34be68> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:189)
"Reference Handler" daemon prio=5 tid=0x00007fd88905f000 nid=0x3703 in
Object.wait() [0x000000011589c000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007cd34b9c8> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:503)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133)
- locked <0x00000007cd34b9c8> (a java.lang.ref.Reference$Lock)
"VM Thread" prio=5 tid=0x00007fd88905e800 nid=0x3503 runnable
"GC task thread#0 (ParallelGC)" prio=5 tid=0x00007fd88b00d000 nid=0x2503
runnable
"GC task thread#1 (ParallelGC)" prio=5 tid=0x00007fd88b00e000 nid=0x2703
runnable
"GC task thread#2 (ParallelGC)" prio=5 tid=0x00007fd88b00e800 nid=0x2903
runnable
"GC task thread#3 (ParallelGC)" prio=5 tid=0x00007fd88b00f000 nid=0x2b03
runnable
"GC task thread#4 (ParallelGC)" prio=5 tid=0x00007fd889012000 nid=0x2d03
runnable
"GC task thread#5 (ParallelGC)" prio=5 tid=0x00007fd889021000 nid=0x2f03
runnable
"GC task thread#6 (ParallelGC)" prio=5 tid=0x00007fd889022000 nid=0x3103
runnable
"GC task thread#7 (ParallelGC)" prio=5 tid=0x00007fd889022800 nid=0x3303
runnable
"VM Periodic Task Thread" prio=5 tid=0x00007fd889064000 nid=0x5503 waiting on
condition
JNI global references: 156
{code}
> MiniTezCluster takes 10 minutes to shut down
> --------------------------------------------
>
> Key: TEZ-1827
> URL: https://issues.apache.org/jira/browse/TEZ-1827
> Project: Apache Tez
> Issue Type: Bug
> Affects Versions: 0.5.2
> Reporter: Ashutosh Chauhan
> Priority: Minor
>
> Always reproducible
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
