Mohamed Elagamy created SPARK-23659:
---------------------------------------

             Summary: Spark Job gets stuck during shuffle 
                 Key: SPARK-23659
                 URL: https://issues.apache.org/jira/browse/SPARK-23659
             Project: Spark
          Issue Type: Question
          Components: Input/Output
    Affects Versions: 2.2.0
            Reporter: Mohamed Elagamy


Hello Team, 

 

   I am running a standalone spark cluster that has 42 nodes each one is 
considered and executer with 100gb of memory and running an application on it 
that cluster to do some aggregation, the application get's stuck randomly and 
that the thread sump, any guidelines would be highly appreciated, thanks 
community 



Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.60-b23 mixed mode):

"Attach Listener" #464538 daemon prio=9 os_prio=0 tid=0x00007f2948002800 
nid=0x63b8 waiting on condition [0x0000000000000000]
 java.lang.Thread.State: RUNNABLE

Locked ownable synchronizers:
 - None

"block-manager-slave-async-thread-pool-6374" #464493 daemon prio=5 os_prio=0 
tid=0x00007f28005e6000 nid=0x62ea waiting on condition [0x00007f2810de8000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1ceb2b8> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"block-manager-slave-async-thread-pool-6373" #464492 daemon prio=5 os_prio=0 
tid=0x00007f2800023000 nid=0x62e9 waiting on condition [0x00007f2812efd000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1ceb2b8> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"block-manager-slave-async-thread-pool-6372" #464491 daemon prio=5 os_prio=0 
tid=0x00007f28005e7800 nid=0x62e8 waiting on condition [0x00007f27c19f0000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1ceb2b8> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"block-manager-slave-async-thread-pool-6371" #464490 daemon prio=5 os_prio=0 
tid=0x00007f28fc007800 nid=0x62e7 waiting on condition [0x00007f27c67fc000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1ceb2b8> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"Executor task launch worker for task 23543209" #464464 daemon prio=5 os_prio=0 
tid=0x00007f28c4057000 nid=0x62c7 runnable [0x00007f2811ef1000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af0912a78> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"Executor task launch worker for task 23543178" #464463 daemon prio=5 os_prio=0 
tid=0x00007f28c404d800 nid=0x62c6 waiting on condition [0x00007f2969aa2000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af0912a78> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"Executor task launch worker for task 23544415" #464462 daemon prio=5 os_prio=0 
tid=0x00007f28c4027000 nid=0x62c5 waiting on condition [0x00007f27c73fe000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af0912a78> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"Executor task launch worker for task 23544991" #464461 daemon prio=5 os_prio=0 
tid=0x00007f28f4003000 nid=0x62c4 waiting on condition [0x00007f27c66fb000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af0912a78> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"Executor task launch worker for task 23544272" #464460 daemon prio=5 os_prio=0 
tid=0x00007f28e8005000 nid=0x62c3 waiting on condition [0x00007f27c1bf2000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af0912a78> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"Executor task launch worker for task 23543012" #464458 daemon prio=5 os_prio=0 
tid=0x00007f2908272800 nid=0x62c1 runnable [0x00007f28385eb000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af0912a78> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"Executor task launch worker for task 23542978" #464457 daemon prio=5 os_prio=0 
tid=0x00007f28f000a000 nid=0x62c0 waiting on condition [0x00007f289eaf8000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af0912a78> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"Executor task launch worker for task 23544889" #464456 daemon prio=5 os_prio=0 
tid=0x00007f28f0009800 nid=0x62bf waiting on condition [0x00007f27c7bfe000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af0912a78> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"Executor task launch worker for task 23544340" #464455 daemon prio=5 os_prio=0 
tid=0x00007f28f8009000 nid=0x62be waiting on condition [0x00007f2969ea6000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af0912a78> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"Executor task launch worker for task 23544957" #464454 daemon prio=5 os_prio=0 
tid=0x00007f2904002000 nid=0x62bd waiting on condition [0x00007f27c23f4000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af0912a78> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"Executor task launch worker for task 23542812" #464453 daemon prio=5 os_prio=0 
tid=0x00007f2900002000 nid=0x62bc runnable [0x00007f28387ed000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af0912a78> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"Executor task launch worker for task 23542778" #464452 daemon prio=5 os_prio=0 
tid=0x00007f290c002000 nid=0x62bb waiting on condition [0x00007f27c6bfe000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af0912a78> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"Executor task launch worker for task 23544923" #464451 daemon prio=5 os_prio=0 
tid=0x00007f290c001000 nid=0x62ba waiting on condition [0x00007f27c5df8000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af0912a78> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"Executor task launch worker for task 23545025" #464450 daemon prio=5 os_prio=0 
tid=0x00007f28fc006800 nid=0x62b9 waiting on condition [0x00007f27c34ed000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af0912a78> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"LeaseRenewer:portal-admin@10.51.169.68:9011" #464434 daemon prio=5 os_prio=0 
tid=0x00007f287c51b000 nid=0x627c sleeping[0x00007f289ecfa000]
 java.lang.Thread.State: TIMED_WAITING (sleeping)
 at java.lang.Thread.sleep(Native Method)
 at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:438)
 at org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:71)
 at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:298)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"Executor task launch worker for task 23544461" #464432 daemon prio=5 os_prio=0 
tid=0x00007f28b8003000 nid=0x627a waiting on condition [0x00007f29699a1000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af0912a78> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"IPC Parameter Sending Thread #1010" #464429 daemon prio=5 os_prio=0 
tid=0x00007f287c498800 nid=0x6277 waiting on condition [0x00007f289e4f4000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1a8f4a0> (a 
java.util.concurrent.SynchronousQueue$TransferStack)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
 at 
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
 at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"Executor task launch worker for task 23539660" #464239 daemon prio=5 os_prio=0 
tid=0x00007f28f4002000 nid=0x5594 runnable [0x00007f27c5bf5000]
 java.lang.Thread.State: RUNNABLE
 at com.wd.perf.collector.spark.utils.Stats$.merge$1(Stats.scala:35)
 at com.wd.perf.collector.spark.utils.Stats$.mergeSorted(Stats.scala:52)
 at com.wd.perf.collector.spark.utils.Stats$.mergeHistogram(Stats.scala:103)
 at 
com.wd.perf.collector.spark.aggregates.ArrayBasedDoubleHistogramAggregate.$plus(ArrayBasedDoubleHistogramAggregate.scala:23)
 at 
com.wd.perf.collector.spark.aggregates.ArrayBasedDoubleHistogramAggregate$$anonfun$applyOnUnsorted$1.apply(ArrayBasedDoubleHistogramAggregate.scala:133)
 at 
com.wd.perf.collector.spark.aggregates.ArrayBasedDoubleHistogramAggregate$$anonfun$applyOnUnsorted$1.apply(ArrayBasedDoubleHistogramAggregate.scala:132)
 at 
scala.collection.IndexedSeqOptimized$class.foldl(IndexedSeqOptimized.scala:57)
 at 
scala.collection.IndexedSeqOptimized$class.foldLeft(IndexedSeqOptimized.scala:66)
 at scala.collection.mutable.ArrayBuffer.foldLeft(ArrayBuffer.scala:48)
 at 
com.wd.perf.collector.spark.aggregates.ArrayBasedDoubleHistogramAggregate$.applyOnUnsorted(ArrayBasedDoubleHistogramAggregate.scala:132)
 at 
com.wd.perf.collector.spark.aggregates.ArrayBasedDoubleHistogramAggregate$$anonfun$5.apply(ArrayBasedDoubleHistogramAggregate.scala:89)
 at 
com.wd.perf.collector.spark.aggregates.ArrayBasedDoubleHistogramAggregate$$anonfun$5.apply(ArrayBasedDoubleHistogramAggregate.scala:88)
 at 
org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIterator.processNext(Unknown
 Source)
 at 
org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
 at 
org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$8$$anon$1.hasNext(WholeStageCodegenExec.scala:395)
 at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)
 at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)
 at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)
 at 
org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:191)
 at 
org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)
 at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)
 at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)
 at org.apache.spark.scheduler.Task.run(Task.scala:108)
 at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:335)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - <0x00007f2c90ff39b0> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"shuffle-client-4-8" #83 daemon prio=5 os_prio=0 tid=0x00007f287c03d000 
nid=0x74ff runnable [0x00007f2838aee000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af27dd508> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af27e15c0> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af27dd420> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"shuffle-client-4-7" #82 daemon prio=5 os_prio=0 tid=0x00007f287c03b000 
nid=0x74fe runnable [0x00007f2838bef000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af27f40c0> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af27f8178> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af27f3fd8> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"shuffle-client-4-6" #81 daemon prio=5 os_prio=0 tid=0x00007f287c039800 
nid=0x74fd runnable [0x00007f2839cf1000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af280ac78> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af280ed30> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af280ab90> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"shuffle-client-4-5" #80 daemon prio=5 os_prio=0 tid=0x00007f287c039000 
nid=0x74fc runnable [0x00007f283adf3000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af2811818> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af28158d0> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af2811730> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"shuffle-client-4-4" #79 daemon prio=5 os_prio=0 tid=0x00007f287c032000 
nid=0x74fb runnable [0x00007f283aef4000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af28183b8> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af281c470> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af28182d0> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"shuffle-server-5-8" #91 daemon prio=5 os_prio=0 tid=0x00007f2894009800 
nid=0x74fa runnable [0x00007f283cff7000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af1baa8c8> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af1bae980> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af1baa7e0> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"shuffle-server-5-7" #90 daemon prio=5 os_prio=0 tid=0x00007f2894009000 
nid=0x74f9 runnable [0x00007f2968887000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af1bb1468> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af1bb5520> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af1bb1380> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"shuffle-server-5-6" #89 daemon prio=5 os_prio=0 tid=0x00007f2894006000 
nid=0x74f8 runnable [0x00007f2968184000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af256eac8> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af2572b80> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af256e9e0> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"shuffle-server-5-5" #88 daemon prio=5 os_prio=0 tid=0x00007f2894005800 
nid=0x74f7 runnable [0x00007f2858f15000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af2585680> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af2589738> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af2585598> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"shuffle-server-5-4" #87 daemon prio=5 os_prio=0 tid=0x00007f2894003800 
nid=0x74f6 runnable [0x00007f296908d000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af259c238> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af25a02f0> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af259c150> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"threadDeathWatcher-8-1" #228 daemon prio=1 os_prio=0 tid=0x00007f2850003800 
nid=0x74f5 sleeping[0x00007f2969fa7000]
 java.lang.Thread.State: TIMED_WAITING (sleeping)
 at java.lang.Thread.sleep(Native Method)
 at io.netty.util.ThreadDeathWatcher$Watcher.run(ThreadDeathWatcher.java:152)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"shuffle-server-5-3" #86 daemon prio=5 os_prio=0 tid=0x00007f2894004800 
nid=0x74f4 runnable [0x00007f2969ba3000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af25b2df0> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af25b6ea8> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af25b2d08> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"shuffle-server-5-2" #85 daemon prio=5 os_prio=0 tid=0x00007f2894002000 
nid=0x74f3 runnable [0x00007f2858e14000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af29c1e80> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af29c5f38> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af29c1d98> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"shuffle-client-4-3" #78 daemon prio=5 os_prio=0 tid=0x00007f28d0018000 
nid=0x74f2 runnable [0x00007f2858d13000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af2d90430> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af2d944e8> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af2d90348> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"org.apache.hadoop.hdfs.PeerCache@7394064c" #211 daemon prio=5 os_prio=0 
tid=0x00007f286c063800 nid=0x74b3 sleeping[0x00007f2858a12000]
 java.lang.Thread.State: TIMED_WAITING (sleeping)
 at java.lang.Thread.sleep(Native Method)
 at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255)
 at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
 at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"shuffle-client-4-2" #77 daemon prio=5 os_prio=0 tid=0x00007f28d004c000 
nid=0x74af runnable [0x00007f285aa37000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af1dd6a60> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af1dd6a90> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af1dd69e8> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"shuffle-client-4-1" #76 daemon prio=5 os_prio=0 tid=0x00007f288402b800 
nid=0x74ae runnable [0x00007f289d1ee000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af1c0dba0> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af1c0dbd0> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af1c0db28> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"files-client-6-1" #113 daemon prio=5 os_prio=0 tid=0x00007f28d4053800 
nid=0x73f1 runnable [0x00007f289e2f0000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af1b27f70> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af1b27f58> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af1b27fa0> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"driver-heartbeater" #92 daemon prio=5 os_prio=0 tid=0x00007f2908274800 
nid=0x73b2 runnable [0x00007f289eefc000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1b75cb8> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
 at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
 at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"shuffle-server-5-1" #84 daemon prio=5 os_prio=0 tid=0x00007f290824a800 
nid=0x73b1 runnable [0x00007f289effd000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af1a90690> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af1a906c0> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af1a90618> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"rpc-client-3-2" #65 daemon prio=5 os_prio=0 tid=0x00007f28a4001000 nid=0x73af 
runnable [0x00007f2954167000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af1d8bc58> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af1d8bc88> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af1d8bbe0> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"rpc-client-3-1" #64 daemon prio=5 os_prio=0 tid=0x00007f28a8001000 nid=0x73ad 
runnable [0x00007f2968685000]
 java.lang.Thread.State: RUNNABLE
 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
 - locked <0x00007f2af1b29130> (a io.netty.channel.nio.SelectedSelectionKeySet)
 - locked <0x00007f2af1b29160> (a java.util.Collections$UnmodifiableSet)
 - locked <0x00007f2af1b290b8> (a sun.nio.ch.EPollSelectorImpl)
 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
 at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:747)
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:392)
 at 
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
 at 
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"netty-rpc-env-timeout" #73 daemon prio=5 os_prio=0 tid=0x00007f4411477800 
nid=0x73ac runnable [0x00007f2968786000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1c59a78> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
 at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
 at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
 at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - None

"dispatcher-event-loop-15" #63 daemon prio=5 os_prio=0 tid=0x00007f441138f800 
nid=0x73aa waiting on condition [0x00007f2968988000]
 java.lang.Thread.State: WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1bb9778> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
 at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:207)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - <0x00007f2af1bb98d8> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"dispatcher-event-loop-14" #62 daemon prio=5 os_prio=0 tid=0x00007f441138d800 
nid=0x73a9 waiting on condition [0x00007f2968a89000]
 java.lang.Thread.State: WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1bb9778> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
 at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:207)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - <0x00007f2af1d3f660> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"dispatcher-event-loop-13" #61 daemon prio=5 os_prio=0 tid=0x00007f441138c000 
nid=0x73a8 waiting on condition [0x00007f2968b8a000]
 java.lang.Thread.State: WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1bb9778> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
 at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:207)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - <0x00007f2af1b76028> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"dispatcher-event-loop-12" #60 daemon prio=5 os_prio=0 tid=0x00007f441138a000 
nid=0x73a7 waiting on condition [0x00007f2968c8b000]
 java.lang.Thread.State: WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1bb9778> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
 at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:207)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - <0x00007f2af1ca5df8> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"dispatcher-event-loop-11" #59 daemon prio=5 os_prio=0 tid=0x00007f4411388000 
nid=0x73a6 waiting on condition [0x00007f2968d8c000]
 java.lang.Thread.State: WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1bb9778> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
 at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:207)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - <0x00007f2af1d8bfa8> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"dispatcher-event-loop-10" #58 daemon prio=5 os_prio=0 tid=0x00007f4411387000 
nid=0x73a5 waiting on condition [0x00007f296918e000]
 java.lang.Thread.State: WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1bb9778> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
 at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:207)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - <0x00007f2af1a909e0> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"dispatcher-event-loop-9" #57 daemon prio=5 os_prio=0 tid=0x00007f4411386000 
nid=0x73a4 waiting on condition [0x00007f296acb1000]
 java.lang.Thread.State: WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1bb9778> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
 at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:207)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - <0x00007f2af1b29480> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"dispatcher-event-loop-8" #56 daemon prio=5 os_prio=0 tid=0x00007f4411385000 
nid=0x73a3 waiting on condition [0x00007f296a7af000]
 java.lang.Thread.State: WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1bb9778> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
 at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:207)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - <0x00007f2af1c59de8> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"dispatcher-event-loop-7" #55 daemon prio=5 os_prio=0 tid=0x00007f4411384000 
nid=0x73a2 waiting on condition [0x00007f296a6ae000]
 java.lang.Thread.State: WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1bb9778> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
 at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:207)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - <0x00007f2af1ceb938> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"dispatcher-event-loop-6" #54 daemon prio=5 os_prio=0 tid=0x00007f4411383000 
nid=0x73a1 waiting on condition [0x00007f296a5ad000]
 java.lang.Thread.State: WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1bb9778> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
 at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:207)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - <0x00007f2af1bb9b28> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"dispatcher-event-loop-5" #53 daemon prio=5 os_prio=0 tid=0x00007f4411382000 
nid=0x73a0 waiting on condition [0x00007f296a4ac000]
 java.lang.Thread.State: WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1bb9778> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
 at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:207)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - <0x00007f2af1b76278> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"dispatcher-event-loop-4" #52 daemon prio=5 os_prio=0 tid=0x00007f4411381000 
nid=0x739f waiting on condition [0x00007f296a3ab000]
 java.lang.Thread.State: WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1bb9778> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
 at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:207)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - <0x00007f2af1d3f8b0> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"dispatcher-event-loop-3" #51 daemon prio=5 os_prio=0 tid=0x00007f441137f800 
nid=0x739e waiting on condition [0x00007f296a2aa000]
 java.lang.Thread.State: WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1bb9778> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
 at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:207)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - <0x00007f2af1ca6048> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"dispatcher-event-loop-2" #50 daemon prio=5 os_prio=0 tid=0x00007f441137e800 
nid=0x739d waiting on condition [0x00007f296a1a9000]
 java.lang.Thread.State: WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1bb9778> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
 at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:207)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - <0x00007f2af1a90c30> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"dispatcher-event-loop-1" #49 daemon prio=5 os_prio=0 tid=0x00007f441137e000 
nid=0x739c waiting on condition [0x00007f296a0a8000]
 java.lang.Thread.State: WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1bb9778> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
 at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:207)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - <0x00007f2af1bb9d78> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"dispatcher-event-loop-0" #48 daemon prio=5 os_prio=0 tid=0x00007f4411377000 
nid=0x739b waiting on condition [0x00007f296928f000]
 java.lang.Thread.State: WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1bb9778> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
 at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:207)
 at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
 at java.lang.Thread.run(Thread.java:745)

Locked ownable synchronizers:
 - <0x00007f2af1cebb88> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"Service Thread" #18 daemon prio=9 os_prio=0 tid=0x00007f4410356800 nid=0x737c 
runnable [0x0000000000000000]
 java.lang.Thread.State: RUNNABLE

Locked ownable synchronizers:
 - None

"C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f4410351000 
nid=0x737b waiting on condition [0x0000000000000000]
 java.lang.Thread.State: RUNNABLE

Locked ownable synchronizers:
 - None

"C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f441034f800 
nid=0x737a waiting on condition [0x0000000000000000]
 java.lang.Thread.State: RUNNABLE

Locked ownable synchronizers:
 - None

"C1 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f441034c800 
nid=0x7379 waiting on condition [0x0000000000000000]
 java.lang.Thread.State: RUNNABLE

Locked ownable synchronizers:
 - None

"C1 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f441034a800 
nid=0x7378 runnable [0x0000000000000000]
 java.lang.Thread.State: RUNNABLE

Locked ownable synchronizers:
 - None

"C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f4410348000 
nid=0x7377 waiting on condition [0x0000000000000000]
 java.lang.Thread.State: RUNNABLE

Locked ownable synchronizers:
 - None

"C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f4410346000 
nid=0x7376 waiting on condition [0x0000000000000000]
 java.lang.Thread.State: RUNNABLE

Locked ownable synchronizers:
 - None

"C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f4410344000 
nid=0x7375 waiting on condition [0x0000000000000000]
 java.lang.Thread.State: RUNNABLE

Locked ownable synchronizers:
 - None

"C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f4410341800 
nid=0x7374 waiting on condition [0x0000000000000000]
 java.lang.Thread.State: RUNNABLE

Locked ownable synchronizers:
 - None

"C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f4410337800 
nid=0x7373 waiting on condition [0x0000000000000000]
 java.lang.Thread.State: RUNNABLE

Locked ownable synchronizers:
 - None

"C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f4410335000 
nid=0x7372 waiting on condition [0x0000000000000000]
 java.lang.Thread.State: RUNNABLE

Locked ownable synchronizers:
 - None

"C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f4410333000 
nid=0x7371 waiting on condition [0x0000000000000000]
 java.lang.Thread.State: RUNNABLE

Locked ownable synchronizers:
 - None

"C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f4410330800 
nid=0x7370 waiting on condition [0x0000000000000000]
 java.lang.Thread.State: RUNNABLE

Locked ownable synchronizers:
 - None

"Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f441032e800 
nid=0x736f runnable [0x0000000000000000]
 java.lang.Thread.State: RUNNABLE

Locked ownable synchronizers:
 - None

"Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 
tid=0x00007f441032d000 nid=0x736e waiting on condition [0x0000000000000000]
 java.lang.Thread.State: RUNNABLE

Locked ownable synchronizers:
 - None

"Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f44102f7000 nid=0x736d in 
Object.wait() [0x00007f43f8286000]
 java.lang.Thread.State: WAITING (on object monitor)
 at java.lang.Object.wait(Native Method)
 at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
 - locked <0x00007f2af1c5a1a0> (a java.lang.ref.ReferenceQueue$Lock)
 at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
 at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)

Locked ownable synchronizers:
 - None

"Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f44102f5000 
nid=0x736c in Object.wait() [0x00007f43f8387000]
 java.lang.Thread.State: WAITING (on object monitor)
 at java.lang.Object.wait(Native Method)
 at java.lang.Object.wait(Object.java:502)
 at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:157)
 - locked <0x00007f2af1dd7570> (a java.lang.ref.Reference$Lock)

Locked ownable synchronizers:
 - None

"main" #1 prio=5 os_prio=0 tid=0x00007f441000d800 nid=0x734b waiting on 
condition [0x00007f4416a53000]
 java.lang.Thread.State: TIMED_WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00007f2af1cebf08> (a 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
 at 
java.util.concurrent.ThreadPoolExecutor.awaitTermination(ThreadPoolExecutor.java:1465)
 at org.apache.spark.rpc.netty.Dispatcher.awaitTermination(Dispatcher.scala:180)
 at 
org.apache.spark.rpc.netty.NettyRpcEnv.awaitTermination(NettyRpcEnv.scala:281)
 at 
org.apache.spark.executor.CoarseGrainedExecutorBackend$$anonfun$run$1.apply$mcV$sp(CoarseGrainedExecutorBackend.scala:231)
 at 
org.apache.spark.deploy.SparkHadoopUtil$$anon$2.run(SparkHadoopUtil.scala:67)
 at 
org.apache.spark.deploy.SparkHadoopUtil$$anon$2.run(SparkHadoopUtil.scala:66)
 at java.security.AccessController.doPrivileged(Native Method)
 at javax.security.auth.Subject.doAs(Subject.java:422)
 at 
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
 at 
org.apache.spark.deploy.SparkHadoopUtil.runAsSparkUser(SparkHadoopUtil.scala:66)
 at 
org.apache.spark.executor.CoarseGrainedExecutorBackend$.run(CoarseGrainedExecutorBackend.scala:188)
 at 
org.apache.spark.executor.CoarseGrainedExecutorBackend$.main(CoarseGrainedExecutorBackend.scala:284)
 at 
org.apache.spark.executor.CoarseGrainedExecutorBackend.main(CoarseGrainedExecutorBackend.scala)

Locked ownable synchronizers:
 - None

"VM Thread" os_prio=0 tid=0x00007f44102ef800 nid=0x736b runnable

"Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f4410025800 
nid=0x734c runnable

"Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f4410027000 
nid=0x734d runnable

"Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f4410029000 
nid=0x734e runnable

"Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f441002b000 
nid=0x734f runnable

"Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f441002c800 
nid=0x7350 runnable

"Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f441002e800 
nid=0x7351 runnable

"Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f4410030000 
nid=0x7352 runnable

"Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f4410032000 
nid=0x7353 runnable

"Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f4410034000 
nid=0x7354 runnable

"Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f4410035800 
nid=0x7355 runnable

"Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f4410037800 
nid=0x7356 runnable

"Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f4410039800 
nid=0x7357 runnable

"Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f441003b000 
nid=0x7358 runnable

"G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f441008a000 nid=0x7367 
runnable

"Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f441008b800 
nid=0x7368 runnable

"Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f441008d800 
nid=0x7369 runnable

"Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f441008f800 
nid=0x736a runnable

"G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f4410056000 nid=0x7366 
runnable

"G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f4410054800 nid=0x7365 
runnable

"G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f4410052800 nid=0x7364 
runnable

"G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f4410050800 nid=0x7363 
runnable

"G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f441004f000 nid=0x7362 
runnable

"G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f441004d000 nid=0x7361 
runnable

"G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f441004b000 nid=0x7360 
runnable

"G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f4410049800 nid=0x735f 
runnable

"G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f4410047800 nid=0x735e 
runnable

"G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f4410045800 nid=0x735d 
runnable

"G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f4410044000 
nid=0x735c runnable

"G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f4410042000 
nid=0x735b runnable

"G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f4410040000 
nid=0x735a runnable

"G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f441003e000 
nid=0x7359 runnable

"VM Periodic Task Thread" os_prio=0 tid=0x00007f4410359800 nid=0x737d waiting 
on condition



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org

Reply via email to