Added this JIRA https://issues.apache.org/jira/browse/FLINK-5993 to track this. Would be great to comment there if you have any other issues that should be covered in a Azure deployment section.
On Tue, Mar 7, 2017 at 7:10 PM, Stephan Ewen <se...@apache.org> wrote: > Great to hear it! > > What do you think about adding a section to the Flink docs about deployment > on Azure (there is already AWS and GCE, so Azure would make the > cloud-trinity complete) that explains how to set this up and avoid such > pitfalls. > > https://ci.apache.org/projects/flink/flink-docs-release-1.2/ > > If there is anything you think can be improved in the dependencies to help > there, please let us know! > > > On Tue, Mar 7, 2017 at 5:31 PM, Avihai Berkovitz > <avihai.berkov...@microsoft.com> wrote: >> >> For anyone seeing this thread in the future, we managed to solve the >> issue. The problem was in the Azure storage SDK. >> >> Flink is using Hadoop 2.7, so we used version 2.7.3 of the Hadoop-azure >> package. This package uses version 2.0.0 of the azure-storage package, dated >> from 2014. It has several bugs that were since fixed, specifically one where >> the socket timeout was infinite. We updated this package to version 5.0.0 >> and everything is working smoothly now. >> >> >> >> From: Stephan Ewen [mailto:se...@apache.org] >> Sent: Sunday, February 26, 2017 4:47 PM >> To: user@flink.apache.org >> Subject: Re: Flink checkpointing gets stuck >> >> >> >> Thanks! >> >> >> >> This looks like a bigger example, involving MongoDB, etc. >> >> Are you able to reproduce this issue with a smaller example? >> >> >> >> It would also help to understand the problem better if we knew the >> topology a bit better. >> >> The stack traces look like "phase 1&2" want to send data (but are back >> pressured) and "phase 3&4&5" wait for input data. >> >> >> >> Stephan >> >> >> >> >> >> On Sun, Feb 26, 2017 at 12:30 PM, Shai Kaplan <shai.kap...@microsoft.com> >> wrote: >> >> Running jstack on one of the Task Managers: >> >> >> >> 2017-02-26 10:06:27 >> >> Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.111-b14 mixed >> mode): >> >> >> >> "Attach Listener" #6414 daemon prio=9 os_prio=0 tid=0x00007f3c8c089000 >> nid=0xe692 waiting on condition [0x0000000000000000] >> >> java.lang.Thread.State: RUNNABLE >> >> >> >> "Async calls on Sink: phase 5 (32/48)" #2337 daemon prio=5 os_prio=0 >> tid=0x00007f3b942fc000 nid=0xb0d5 waiting on condition [0x00007f3adf0af000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9d000620> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on Sink: phase 5 (31/48)" #2336 daemon prio=5 os_prio=0 >> tid=0x00007f3b942fb000 nid=0xb0d4 waiting on condition [0x00007f3adf1b0000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9fbd7e70> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on Sink: phase 5 (30/48)" #2335 daemon prio=5 os_prio=0 >> tid=0x00007f3b942f9800 nid=0xb0d3 waiting on condition [0x00007f3adf2b1000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3da07cdde8> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on Sink: phase 5 (29/48)" #2334 daemon prio=5 os_prio=0 >> tid=0x00007f3b942f8800 nid=0xb0d2 waiting on condition [0x00007f3adf3b2000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9e0003e8> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on Sink: phase 5 (28/48)" #2333 daemon prio=5 os_prio=0 >> tid=0x00007f3b942f7000 nid=0xb0d1 waiting on condition [0x00007f3adf4b3000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9fbd8008> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on Sink: phase 5 (27/48)" #2332 daemon prio=5 os_prio=0 >> tid=0x00007f3b942f6000 nid=0xb0d0 waiting on condition [0x00007f3adf5b4000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9e000730> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on Sink: phase 5 (26/48)" #2331 daemon prio=5 os_prio=0 >> tid=0x00007f3b942f4800 nid=0xb0cf waiting on condition [0x00007f3adf6b5000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3da07cdf80> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on Sink: phase 5 (25/48)" #2330 daemon prio=5 os_prio=0 >> tid=0x00007f3b942f3000 nid=0xb0ce waiting on condition [0x00007f3adf7b6000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9e000a78> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on Sink: phase 5 (24/48)" #2329 daemon prio=5 os_prio=0 >> tid=0x00007f3b942f1800 nid=0xb0cd waiting on condition [0x00007f3adf8b7000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9fbd81a0> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on Sink: phase 5 (23/48)" #2328 daemon prio=5 os_prio=0 >> tid=0x00007f3b942f0800 nid=0xb0cc waiting on condition [0x00007f3adf9b8000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9d000968> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on Sink: phase 5 (22/48)" #2327 daemon prio=5 os_prio=0 >> tid=0x00007f3b942ef000 nid=0xb0cb waiting on condition [0x00007f3adfab9000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3da07ce118> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on Sink: phase 5 (21/48)" #2326 daemon prio=5 os_prio=0 >> tid=0x00007f3b942ee000 nid=0xb0ca waiting on condition [0x00007f3adfbba000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9fbd8338> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on Sink: phase 5 (20/48)" #2325 daemon prio=5 os_prio=0 >> tid=0x00007f3b942ec800 nid=0xb0c9 waiting on condition [0x00007f3adfcbb000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9d000cb0> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on Sink: phase 5 (19/48)" #2324 daemon prio=5 os_prio=0 >> tid=0x00007f3b942eb000 nid=0xb0c8 waiting on condition [0x00007f3adfdbc000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3da07ce2b0> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on Sink: phase 5 (18/48)" #2323 daemon prio=5 os_prio=0 >> tid=0x00007f3b942e9800 nid=0xb0c7 waiting on condition [0x00007f3adfebd000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9fbd84d0> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on Sink: phase 5 (17/48)" #2322 daemon prio=5 os_prio=0 >> tid=0x00007f3b942e8800 nid=0xb0c6 waiting on condition [0x00007f3adffbe000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9e000dc0> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on phase 4 (32/48)" #2321 daemon prio=5 os_prio=0 >> tid=0x00007f3b94257800 nid=0xb0c5 waiting on condition [0x00007f3ae00bf000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9d000ff8> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on phase 4 (31/48)" #2320 daemon prio=5 os_prio=0 >> tid=0x00007f3b94256000 nid=0xb0c4 waiting on condition [0x00007f3ae01c0000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3da07ce448> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on phase 4 (30/48)" #2319 daemon prio=5 os_prio=0 >> tid=0x00007f3b94254800 nid=0xb0c3 waiting on condition [0x00007f3ae02c1000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9fbd8668> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on phase 4 (29/48)" #2318 daemon prio=5 os_prio=0 >> tid=0x00007f3b94253000 nid=0xb0c2 waiting on condition [0x00007f3ae03c2000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9d001340> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on phase 4 (28/48)" #2317 daemon prio=5 os_prio=0 >> tid=0x00007f3b94251800 nid=0xb0c1 waiting on condition [0x00007f3ae04c3000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9e001108> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on phase 4 (27/48)" #2316 daemon prio=5 os_prio=0 >> tid=0x00007f3b94250000 nid=0xb0c0 waiting on condition [0x00007f3ae06c5000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3da07ce5e0> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on phase 4 (26/48)" #2315 daemon prio=5 os_prio=0 >> tid=0x00007f3b9424e800 nid=0xb0bf waiting on condition [0x00007f3ae09c8000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9fbd8800> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on phase 4 (25/48)" #2314 daemon prio=5 os_prio=0 >> tid=0x00007f3b9424d800 nid=0xb0be waiting on condition [0x00007f3ae0ccb000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9e001450> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on phase 4 (24/48)" #2313 daemon prio=5 os_prio=0 >> tid=0x00007f3b9424c000 nid=0xb0bd waiting on condition [0x00007f3ae0dcc000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9d001688> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on phase 4 (23/48)" #2312 daemon prio=5 os_prio=0 >> tid=0x00007f3b9424a800 nid=0xb0bc waiting on condition [0x00007f3ae0ecd000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9e001798> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on phase 4 (22/48)" #2311 daemon prio=5 os_prio=0 >> tid=0x00007f3b94249000 nid=0xb0bb waiting on condition [0x00007f3ae0fce000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9d0019d0> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on phase 4 (21/48)" #2310 daemon prio=5 os_prio=0 >> tid=0x00007f3b94247800 nid=0xb0ba waiting on condition [0x00007f3ae31f0000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9fbd8998> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on phase 4 (20/48)" #2309 daemon prio=5 os_prio=0 >> tid=0x00007f3b94246800 nid=0xb0b9 waiting on condition [0x00007f3ae3cfb000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3da07ce778> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on phase 4 (19/48)" #2308 daemon prio=5 os_prio=0 >> tid=0x00007f3b94245800 nid=0xb0b8 waiting on condition [0x00007f3ae3efd000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9e001ae0> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on phase 4 (18/48)" #2307 daemon prio=5 os_prio=0 >> tid=0x00007f3b94244000 nid=0xb0b7 waiting on condition [0x00007f3ae3ffe000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9d001d18> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on phase 4 (17/48)" #2306 daemon prio=5 os_prio=0 >> tid=0x00007f3b94242800 nid=0xb0b6 waiting on condition [0x00007f3af02c5000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.park(Native Method) >> >> - parking to wait for <0x00007f3d9fbd8b30> (a >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) >> >> at >> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) >> >> at >> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) >> >> at >> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) >> >> at >> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) >> >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> >> at java.lang.Thread.run(Thread.java:745) >> >> >> >> "Async calls on phase 3 (32/48)" #2305 daemon prio=5 os_prio=0 >> tid=0x00007f3b9423e800 nid=0xb0b5 waiting on condition [0x00007f3af09cc000] >> >> java.lang.Thread.State: WAITING (parking) >> >> at sun.misc.Unsafe.pa >> >> ... >> >> [Message clipped] >> >> > >
