[
https://issues.apache.org/jira/browse/FLINK-19369?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17223039#comment-17223039
]
Matthias edited comment on FLINK-19369 at 10/29/20, 4:58 PM:
-------------------------------------------------------------
And a different probably infinitely running test having the following Thread
stack:
{quote}
2020-10-29 17:54:30
Full thread dump OpenJDK 64-Bit Server VM (11.0.8+10 mixed mode):
Threads class SMR info:
_java_thread_list=0x00007fc0af74b890, length=13, elements={
0x00007fc0b000b800, 0x00007fc0b5808800, 0x00007fc0b580d800, 0x00007fc0b581e000,
0x00007fc0b5811800, 0x00007fc0b581f000, 0x00007fc0b3008800, 0x00007fc0b00f0800,
0x00007fc0b580a000, 0x00007fc0b58cd000, 0x00007fc0b0264000, 0x00007fc0b0a16800,
0x00007fc0b0a75000
}
"main" #1 prio=5 os_prio=31 cpu=3402.11ms elapsed=124.79s
tid=0x00007fc0b000b800 nid=0x1803 runnable [0x000070000763f000]
java.lang.Thread.State: RUNNABLE
at java.net.SocketInputStream.socketRead0([email protected]/Native
Method)
at
java.net.SocketInputStream.socketRead([email protected]/SocketInputStream.java:115)
at
java.net.SocketInputStream.read([email protected]/SocketInputStream.java:168)
at
java.net.SocketInputStream.read([email protected]/SocketInputStream.java:140)
at
java.net.SocketInputStream.read([email protected]/SocketInputStream.java:200)
at
org.apache.flink.runtime.blob.BlobOutputStream.receiveAndCheckPutResponse(BlobOutputStream.java:151)
at
org.apache.flink.runtime.blob.BlobOutputStream.finish(BlobOutputStream.java:104)
at
org.apache.flink.runtime.blob.BlobClient.putBuffer(BlobClient.java:342)
at
org.apache.flink.runtime.blob.BlobClientTest.testGetFailsDuringStreaming(BlobClientTest.java:431)
at
org.apache.flink.runtime.blob.BlobClientTest.testGetFailsDuringStreamingForJobTransientBlob(BlobClientTest.java:404)
at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke([email protected]/DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke([email protected]/Method.java:566)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at
com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:69)
at
com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:52)
at
com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:220)
at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:53)
"Reference Handler" #2 daemon prio=10 os_prio=31 cpu=1.13ms elapsed=124.76s
tid=0x00007fc0b5808800 nid=0x3b03 waiting on condition [0x0000700007d55000]
java.lang.Thread.State: RUNNABLE
at
java.lang.ref.Reference.waitForReferencePendingList([email protected]/Native
Method)
at
java.lang.ref.Reference.processPendingReferences([email protected]/Reference.java:241)
at
java.lang.ref.Reference$ReferenceHandler.run([email protected]/Reference.java:213)
"Finalizer" #3 daemon prio=8 os_prio=31 cpu=0.81ms elapsed=124.76s
tid=0x00007fc0b580d800 nid=0x3d03 in Object.wait() [0x0000700007e58000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait([email protected]/Native Method)
- waiting on <0x000000078084cf08> (a java.lang.ref.ReferenceQueue$Lock)
at
java.lang.ref.ReferenceQueue.remove([email protected]/ReferenceQueue.java:155)
- waiting to re-lock in wait() <0x000000078084cf08> (a
java.lang.ref.ReferenceQueue$Lock)
at
java.lang.ref.ReferenceQueue.remove([email protected]/ReferenceQueue.java:176)
at
java.lang.ref.Finalizer$FinalizerThread.run([email protected]/Finalizer.java:170)
"Signal Dispatcher" #4 daemon prio=9 os_prio=31 cpu=0.26ms elapsed=124.74s
tid=0x00007fc0b581e000 nid=0x4203 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" #5 daemon prio=9 os_prio=31 cpu=1064.32ms elapsed=124.74s
tid=0x00007fc0b5811800 nid=0xa903 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
No compile task
"C1 CompilerThread0" #8 daemon prio=9 os_prio=31 cpu=671.99ms elapsed=124.74s
tid=0x00007fc0b581f000 nid=0x5703 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
No compile task
"Sweeper thread" #9 daemon prio=9 os_prio=31 cpu=20.64ms elapsed=124.74s
tid=0x00007fc0b3008800 nid=0xa503 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Common-Cleaner" #10 daemon prio=8 os_prio=31 cpu=1.18ms elapsed=124.67s
tid=0x00007fc0b00f0800 nid=0xa303 in Object.wait() [0x000070000846d000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait([email protected]/Native Method)
- waiting on <0x000000078084daf8> (a java.lang.ref.ReferenceQueue$Lock)
at
java.lang.ref.ReferenceQueue.remove([email protected]/ReferenceQueue.java:155)
- waiting to re-lock in wait() <0x000000078084daf8> (a
java.lang.ref.ReferenceQueue$Lock)
at
jdk.internal.ref.CleanerImpl.run([email protected]/CleanerImpl.java:148)
at java.lang.Thread.run([email protected]/Thread.java:834)
at
jdk.internal.misc.InnocuousThread.run([email protected]/InnocuousThread.java:134)
"Monitor Ctrl-Break" #11 daemon prio=5 os_prio=31 cpu=16.32ms elapsed=124.62s
tid=0x00007fc0b580a000 nid=0x5b03 runnable [0x0000700008570000]
java.lang.Thread.State: RUNNABLE
at java.net.SocketInputStream.socketRead0([email protected]/Native
Method)
at
java.net.SocketInputStream.socketRead([email protected]/SocketInputStream.java:115)
at
java.net.SocketInputStream.read([email protected]/SocketInputStream.java:168)
at
java.net.SocketInputStream.read([email protected]/SocketInputStream.java:140)
at
sun.nio.cs.StreamDecoder.readBytes([email protected]/StreamDecoder.java:284)
at
sun.nio.cs.StreamDecoder.implRead([email protected]/StreamDecoder.java:326)
at
sun.nio.cs.StreamDecoder.read([email protected]/StreamDecoder.java:178)
- locked <0x000000078084d568> (a java.io.InputStreamReader)
at
java.io.InputStreamReader.read([email protected]/InputStreamReader.java:185)
at java.io.BufferedReader.fill([email protected]/BufferedReader.java:161)
at
java.io.BufferedReader.readLine([email protected]/BufferedReader.java:326)
- locked <0x000000078084d568> (a java.io.InputStreamReader)
at
java.io.BufferedReader.readLine([email protected]/BufferedReader.java:392)
at
com.intellij.rt.execution.application.AppMainV2$1.run(AppMainV2.java:61)
"Service Thread" #12 daemon prio=9 os_prio=31 cpu=0.04ms elapsed=124.62s
tid=0x00007fc0b58cd000 nid=0x5c03 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Attach Listener" #14 daemon prio=9 os_prio=31 cpu=9.18ms elapsed=123.76s
tid=0x00007fc0b0264000 nid=0x6103 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Timer-44" #192 daemon prio=5 os_prio=31 cpu=0.10ms elapsed=120.84s
tid=0x00007fc0b0a16800 nid=0x8e77 in Object.wait() [0x0000700008879000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait([email protected]/Native Method)
- waiting on <0x000000078fe57b50> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop([email protected]/Timer.java:553)
- waiting to re-lock in wait() <0x000000078fe57b50> (a
java.util.TaskQueue)
at java.util.TimerThread.run([email protected]/Timer.java:506)
"BLOB Server listener at 65030" #191 daemon prio=5 os_prio=31 cpu=0.12ms
elapsed=120.84s tid=0x00007fc0b0a75000 nid=0x91b3 runnable [0x0000700008f8e000]
java.lang.Thread.State: RUNNABLE
at java.net.PlainSocketImpl.socketAccept([email protected]/Native Method)
at
java.net.AbstractPlainSocketImpl.accept([email protected]/AbstractPlainSocketImpl.java:458)
at
java.net.ServerSocket.implAccept([email protected]/ServerSocket.java:565)
at java.net.ServerSocket.accept([email protected]/ServerSocket.java:533)
at org.apache.flink.runtime.blob.BlobServer.run(BlobServer.java:262)
"VM Thread" os_prio=31 cpu=31.07ms elapsed=124.77s tid=0x00007fc0b080e000
nid=0x3903 runnable
"GC Thread#0" os_prio=31 cpu=24.78ms elapsed=124.79s tid=0x00007fc0b101b800
nid=0x2e03 runnable
"GC Thread#1" os_prio=31 cpu=23.25ms elapsed=124.28s tid=0x00007fc0b2812000
nid=0x9b03 runnable
"GC Thread#2" os_prio=31 cpu=28.46ms elapsed=124.28s tid=0x00007fc0b2813000
nid=0x9903 runnable
"GC Thread#3" os_prio=31 cpu=21.70ms elapsed=124.28s tid=0x00007fc0afb06800
nid=0x5f03 runnable
"GC Thread#4" os_prio=31 cpu=22.83ms elapsed=124.28s tid=0x00007fc0afb07000
nid=0x9603 runnable
"GC Thread#5" os_prio=31 cpu=22.38ms elapsed=124.28s tid=0x00007fc0afb08000
nid=0x9503 runnable
"G1 Main Marker" os_prio=31 cpu=1.48ms elapsed=124.79s tid=0x00007fc0b0035000
nid=0x3003 runnable
"G1 Conc#0" os_prio=31 cpu=17.64ms elapsed=124.79s tid=0x00007fc0b0035800
nid=0x3303 runnable
"G1 Conc#1" os_prio=31 cpu=20.39ms elapsed=122.77s tid=0x00007fc0b600b000
nid=0x9c07 runnable
"G1 Conc#2" os_prio=31 cpu=18.86ms elapsed=122.77s tid=0x00007fc0b687e800
nid=0x9d07 runnable
"G1 Refine#0" os_prio=31 cpu=0.52ms elapsed=124.78s tid=0x00007fc0b00de800
nid=0x3503 runnable
"G1 Young RemSet Sampling" os_prio=31 cpu=21.50ms elapsed=124.78s
tid=0x00007fc0b00df000 nid=0x5103 runnable
"VM Periodic Task Thread" os_prio=31 cpu=86.53ms elapsed=124.62s
tid=0x00007fc0b108c800 nid=0x9e03 waiting on condition
JNI global refs: 20, weak refs: 0
Heap
garbage-first heap total 262144K, used 65749K [0x0000000780000000,
0x0000000800000000)
region size 1024K, 9 young (9216K), 3 survivors (3072K)
Metaspace used 18385K, capacity 18778K, committed 19072K, reserved
1067008K
class space used 1965K, capacity 2144K, committed 2176K, reserved 1048576K
{quote}
The single test failed after 5mins:
{quote}
java.io.IOException: PUT operation failed: Read timed out
at
org.apache.flink.runtime.blob.BlobClient.putBuffer(BlobClient.java:345)
at
org.apache.flink.runtime.blob.BlobClientTest.testGetFailsDuringStreaming(BlobClientTest.java:431)
at
org.apache.flink.runtime.blob.BlobClientTest.testGetFailsDuringStreamingForJobTransientBlob(BlobClientTest.java:404)
at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at
com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:69)
at
com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:52)
at
com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:220)
at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:53)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.base/java.net.SocketInputStream.socketRead0(Native Method)
at
java.base/java.net.SocketInputStream.socketRead(SocketInputStream.java:115)
at java.base/java.net.SocketInputStream.read(SocketInputStream.java:168)
at java.base/java.net.SocketInputStream.read(SocketInputStream.java:140)
at java.base/java.net.SocketInputStream.read(SocketInputStream.java:200)
at
org.apache.flink.runtime.blob.BlobOutputStream.receiveAndCheckPutResponse(BlobOutputStream.java:151)
at
org.apache.flink.runtime.blob.BlobOutputStream.finish(BlobOutputStream.java:104)
at
org.apache.flink.runtime.blob.BlobClient.putBuffer(BlobClient.java:342)
... 29 more
{quote}
was (Author: mapohl):
And a different probably infinitely running test having the following Thread
stack:
{quote}
2020-10-29 17:54:30
Full thread dump OpenJDK 64-Bit Server VM (11.0.8+10 mixed mode):
Threads class SMR info:
_java_thread_list=0x00007fc0af74b890, length=13, elements={
0x00007fc0b000b800, 0x00007fc0b5808800, 0x00007fc0b580d800, 0x00007fc0b581e000,
0x00007fc0b5811800, 0x00007fc0b581f000, 0x00007fc0b3008800, 0x00007fc0b00f0800,
0x00007fc0b580a000, 0x00007fc0b58cd000, 0x00007fc0b0264000, 0x00007fc0b0a16800,
0x00007fc0b0a75000
}
"main" #1 prio=5 os_prio=31 cpu=3402.11ms elapsed=124.79s
tid=0x00007fc0b000b800 nid=0x1803 runnable [0x000070000763f000]
java.lang.Thread.State: RUNNABLE
at java.net.SocketInputStream.socketRead0([email protected]/Native
Method)
at
java.net.SocketInputStream.socketRead([email protected]/SocketInputStream.java:115)
at
java.net.SocketInputStream.read([email protected]/SocketInputStream.java:168)
at
java.net.SocketInputStream.read([email protected]/SocketInputStream.java:140)
at
java.net.SocketInputStream.read([email protected]/SocketInputStream.java:200)
at
org.apache.flink.runtime.blob.BlobOutputStream.receiveAndCheckPutResponse(BlobOutputStream.java:151)
at
org.apache.flink.runtime.blob.BlobOutputStream.finish(BlobOutputStream.java:104)
at
org.apache.flink.runtime.blob.BlobClient.putBuffer(BlobClient.java:342)
at
org.apache.flink.runtime.blob.BlobClientTest.testGetFailsDuringStreaming(BlobClientTest.java:431)
at
org.apache.flink.runtime.blob.BlobClientTest.testGetFailsDuringStreamingForJobTransientBlob(BlobClientTest.java:404)
at jdk.internal.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke([email protected]/DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke([email protected]/Method.java:566)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at
com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:69)
at
com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:52)
at
com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:220)
at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:53)
"Reference Handler" #2 daemon prio=10 os_prio=31 cpu=1.13ms elapsed=124.76s
tid=0x00007fc0b5808800 nid=0x3b03 waiting on condition [0x0000700007d55000]
java.lang.Thread.State: RUNNABLE
at
java.lang.ref.Reference.waitForReferencePendingList([email protected]/Native
Method)
at
java.lang.ref.Reference.processPendingReferences([email protected]/Reference.java:241)
at
java.lang.ref.Reference$ReferenceHandler.run([email protected]/Reference.java:213)
"Finalizer" #3 daemon prio=8 os_prio=31 cpu=0.81ms elapsed=124.76s
tid=0x00007fc0b580d800 nid=0x3d03 in Object.wait() [0x0000700007e58000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait([email protected]/Native Method)
- waiting on <0x000000078084cf08> (a java.lang.ref.ReferenceQueue$Lock)
at
java.lang.ref.ReferenceQueue.remove([email protected]/ReferenceQueue.java:155)
- waiting to re-lock in wait() <0x000000078084cf08> (a
java.lang.ref.ReferenceQueue$Lock)
at
java.lang.ref.ReferenceQueue.remove([email protected]/ReferenceQueue.java:176)
at
java.lang.ref.Finalizer$FinalizerThread.run([email protected]/Finalizer.java:170)
"Signal Dispatcher" #4 daemon prio=9 os_prio=31 cpu=0.26ms elapsed=124.74s
tid=0x00007fc0b581e000 nid=0x4203 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" #5 daemon prio=9 os_prio=31 cpu=1064.32ms elapsed=124.74s
tid=0x00007fc0b5811800 nid=0xa903 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
No compile task
"C1 CompilerThread0" #8 daemon prio=9 os_prio=31 cpu=671.99ms elapsed=124.74s
tid=0x00007fc0b581f000 nid=0x5703 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
No compile task
"Sweeper thread" #9 daemon prio=9 os_prio=31 cpu=20.64ms elapsed=124.74s
tid=0x00007fc0b3008800 nid=0xa503 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Common-Cleaner" #10 daemon prio=8 os_prio=31 cpu=1.18ms elapsed=124.67s
tid=0x00007fc0b00f0800 nid=0xa303 in Object.wait() [0x000070000846d000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait([email protected]/Native Method)
- waiting on <0x000000078084daf8> (a java.lang.ref.ReferenceQueue$Lock)
at
java.lang.ref.ReferenceQueue.remove([email protected]/ReferenceQueue.java:155)
- waiting to re-lock in wait() <0x000000078084daf8> (a
java.lang.ref.ReferenceQueue$Lock)
at
jdk.internal.ref.CleanerImpl.run([email protected]/CleanerImpl.java:148)
at java.lang.Thread.run([email protected]/Thread.java:834)
at
jdk.internal.misc.InnocuousThread.run([email protected]/InnocuousThread.java:134)
"Monitor Ctrl-Break" #11 daemon prio=5 os_prio=31 cpu=16.32ms elapsed=124.62s
tid=0x00007fc0b580a000 nid=0x5b03 runnable [0x0000700008570000]
java.lang.Thread.State: RUNNABLE
at java.net.SocketInputStream.socketRead0([email protected]/Native
Method)
at
java.net.SocketInputStream.socketRead([email protected]/SocketInputStream.java:115)
at
java.net.SocketInputStream.read([email protected]/SocketInputStream.java:168)
at
java.net.SocketInputStream.read([email protected]/SocketInputStream.java:140)
at
sun.nio.cs.StreamDecoder.readBytes([email protected]/StreamDecoder.java:284)
at
sun.nio.cs.StreamDecoder.implRead([email protected]/StreamDecoder.java:326)
at
sun.nio.cs.StreamDecoder.read([email protected]/StreamDecoder.java:178)
- locked <0x000000078084d568> (a java.io.InputStreamReader)
at
java.io.InputStreamReader.read([email protected]/InputStreamReader.java:185)
at java.io.BufferedReader.fill([email protected]/BufferedReader.java:161)
at
java.io.BufferedReader.readLine([email protected]/BufferedReader.java:326)
- locked <0x000000078084d568> (a java.io.InputStreamReader)
at
java.io.BufferedReader.readLine([email protected]/BufferedReader.java:392)
at
com.intellij.rt.execution.application.AppMainV2$1.run(AppMainV2.java:61)
"Service Thread" #12 daemon prio=9 os_prio=31 cpu=0.04ms elapsed=124.62s
tid=0x00007fc0b58cd000 nid=0x5c03 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Attach Listener" #14 daemon prio=9 os_prio=31 cpu=9.18ms elapsed=123.76s
tid=0x00007fc0b0264000 nid=0x6103 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Timer-44" #192 daemon prio=5 os_prio=31 cpu=0.10ms elapsed=120.84s
tid=0x00007fc0b0a16800 nid=0x8e77 in Object.wait() [0x0000700008879000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait([email protected]/Native Method)
- waiting on <0x000000078fe57b50> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop([email protected]/Timer.java:553)
- waiting to re-lock in wait() <0x000000078fe57b50> (a
java.util.TaskQueue)
at java.util.TimerThread.run([email protected]/Timer.java:506)
"BLOB Server listener at 65030" #191 daemon prio=5 os_prio=31 cpu=0.12ms
elapsed=120.84s tid=0x00007fc0b0a75000 nid=0x91b3 runnable [0x0000700008f8e000]
java.lang.Thread.State: RUNNABLE
at java.net.PlainSocketImpl.socketAccept([email protected]/Native Method)
at
java.net.AbstractPlainSocketImpl.accept([email protected]/AbstractPlainSocketImpl.java:458)
at
java.net.ServerSocket.implAccept([email protected]/ServerSocket.java:565)
at java.net.ServerSocket.accept([email protected]/ServerSocket.java:533)
at org.apache.flink.runtime.blob.BlobServer.run(BlobServer.java:262)
"VM Thread" os_prio=31 cpu=31.07ms elapsed=124.77s tid=0x00007fc0b080e000
nid=0x3903 runnable
"GC Thread#0" os_prio=31 cpu=24.78ms elapsed=124.79s tid=0x00007fc0b101b800
nid=0x2e03 runnable
"GC Thread#1" os_prio=31 cpu=23.25ms elapsed=124.28s tid=0x00007fc0b2812000
nid=0x9b03 runnable
"GC Thread#2" os_prio=31 cpu=28.46ms elapsed=124.28s tid=0x00007fc0b2813000
nid=0x9903 runnable
"GC Thread#3" os_prio=31 cpu=21.70ms elapsed=124.28s tid=0x00007fc0afb06800
nid=0x5f03 runnable
"GC Thread#4" os_prio=31 cpu=22.83ms elapsed=124.28s tid=0x00007fc0afb07000
nid=0x9603 runnable
"GC Thread#5" os_prio=31 cpu=22.38ms elapsed=124.28s tid=0x00007fc0afb08000
nid=0x9503 runnable
"G1 Main Marker" os_prio=31 cpu=1.48ms elapsed=124.79s tid=0x00007fc0b0035000
nid=0x3003 runnable
"G1 Conc#0" os_prio=31 cpu=17.64ms elapsed=124.79s tid=0x00007fc0b0035800
nid=0x3303 runnable
"G1 Conc#1" os_prio=31 cpu=20.39ms elapsed=122.77s tid=0x00007fc0b600b000
nid=0x9c07 runnable
"G1 Conc#2" os_prio=31 cpu=18.86ms elapsed=122.77s tid=0x00007fc0b687e800
nid=0x9d07 runnable
"G1 Refine#0" os_prio=31 cpu=0.52ms elapsed=124.78s tid=0x00007fc0b00de800
nid=0x3503 runnable
"G1 Young RemSet Sampling" os_prio=31 cpu=21.50ms elapsed=124.78s
tid=0x00007fc0b00df000 nid=0x5103 runnable
"VM Periodic Task Thread" os_prio=31 cpu=86.53ms elapsed=124.62s
tid=0x00007fc0b108c800 nid=0x9e03 waiting on condition
JNI global refs: 20, weak refs: 0
Heap
garbage-first heap total 262144K, used 65749K [0x0000000780000000,
0x0000000800000000)
region size 1024K, 9 young (9216K), 3 survivors (3072K)
Metaspace used 18385K, capacity 18778K, committed 19072K, reserved
1067008K
class space used 1965K, capacity 2144K, committed 2176K, reserved 1048576K
{quote}
> BlobClientTest.testGetFailsDuringStreamingForJobPermanentBlob hangs
> -------------------------------------------------------------------
>
> Key: FLINK-19369
> URL: https://issues.apache.org/jira/browse/FLINK-19369
> Project: Flink
> Issue Type: Bug
> Components: Runtime / Coordination, Tests
> Affects Versions: 1.11.0
> Reporter: Dian Fu
> Priority: Major
> Labels: test-stability
>
> https://dev.azure.com/apache-flink/apache-flink/_build/results?buildId=6803&view=logs&j=f0ac5c25-1168-55a5-07ff-0e88223afed9&t=39a61cac-5c62-532f-d2c1-dea450a66708
> {code}
> 2020-09-22T21:40:57.5304615Z "main" #1 prio=5 os_prio=0 cpu=18407.84ms
> elapsed=1969.42s tid=0x00007f0730015800 nid=0x79bd waiting for monitor entry
> [0x00007f07389fb000]
> 2020-09-22T21:40:57.5305080Z java.lang.Thread.State: BLOCKED (on object
> monitor)
> 2020-09-22T21:40:57.5305487Z at
> sun.security.ssl.SSLSocketImpl.duplexCloseOutput([email protected]/SSLSocketImpl.java:541)
> 2020-09-22T21:40:57.5306159Z - waiting to lock <0x000000008661a560> (a
> sun.security.ssl.SSLSocketOutputRecord)
> 2020-09-22T21:40:57.5306545Z at
> sun.security.ssl.SSLSocketImpl.close([email protected]/SSLSocketImpl.java:472)
> 2020-09-22T21:40:57.5307045Z at
> org.apache.flink.runtime.blob.BlobUtils.closeSilently(BlobUtils.java:367)
> 2020-09-22T21:40:57.5307605Z at
> org.apache.flink.runtime.blob.BlobServerConnection.close(BlobServerConnection.java:141)
> 2020-09-22T21:40:57.5308337Z at
> org.apache.flink.runtime.blob.BlobClientTest.testGetFailsDuringStreaming(BlobClientTest.java:443)
> 2020-09-22T21:40:57.5308904Z at
> org.apache.flink.runtime.blob.BlobClientTest.testGetFailsDuringStreamingForJobPermanentBlob(BlobClientTest.java:408)
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
