[
https://issues.apache.org/jira/browse/FLINK-17458?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17101403#comment-17101403
]
Zhu Zhu commented on FLINK-17458:
---------------------------------
I encountered error below when running this test locally for 1000 times.
Seems the actor system and its threads were not properly released, leading to
unhealthy memory status.
It might also be the cause of this JIRA issue.
java.lang.OutOfMemoryError: unable to create new native thread
at java.lang.Thread.start0(Native Method)
at java.lang.Thread.start(Thread.java:717)
at
akka.actor.LightArrayRevolverScheduler.<init>(LightArrayRevolverScheduler.scala:298)
at sun.reflect.GeneratedConstructorAccessor16.newInstance(Unknown
Source)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at
akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(ReflectiveDynamicAccess.scala:33)
at scala.util.Try$.apply(Try.scala:192)
at
akka.actor.ReflectiveDynamicAccess.createInstanceFor(ReflectiveDynamicAccess.scala:28)
at
akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(ReflectiveDynamicAccess.scala:39)
at
akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(ReflectiveDynamicAccess.scala:39)
at scala.util.Success.flatMap(Try.scala:231)
at
akka.actor.ReflectiveDynamicAccess.createInstanceFor(ReflectiveDynamicAccess.scala:39)
at akka.actor.ActorSystemImpl.createScheduler(ActorSystem.scala:925)
at akka.actor.ActorSystemImpl.<init>(ActorSystem.scala:786)
at akka.actor.RobustActorSystem.<init>(RobustActorSystem.scala:41)
at
akka.actor.RobustActorSystem$.internalApply(RobustActorSystem.scala:89)
at akka.actor.RobustActorSystem$.apply(RobustActorSystem.scala:70)
at akka.actor.RobustActorSystem$.create(RobustActorSystem.scala:55)
at
org.apache.flink.runtime.akka.AkkaUtils$.createActorSystem(AkkaUtils.scala:125)
at
org.apache.flink.runtime.akka.AkkaUtils$.createActorSystem(AkkaUtils.scala:113)
at
org.apache.flink.runtime.akka.AkkaUtils$.createLocalActorSystem(AkkaUtils.scala:68)
at
org.apache.flink.runtime.akka.AkkaUtils.createLocalActorSystem(AkkaUtils.scala)
at
org.apache.flink.runtime.rpc.TestingRpcService.<init>(TestingRpcService.java:74)
at
org.apache.flink.runtime.rpc.TestingRpcService.<init>(TestingRpcService.java:67)
at
org.apache.flink.runtime.taskexecutor.TaskSubmissionTestEnvironment$Builder.build(TaskSubmissionTestEnvironment.java:349)
at
org.apache.flink.runtime.taskexecutor.TaskExecutorSubmissionTest.testFailingScheduleOrUpdateConsumers(TaskExecutorSubmissionTest.java:545)
at sun.reflect.GeneratedMethodAccessor35.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
> TaskExecutorSubmissionTest#testFailingScheduleOrUpdateConsumers
> ---------------------------------------------------------------
>
> Key: FLINK-17458
> URL: https://issues.apache.org/jira/browse/FLINK-17458
> Project: Flink
> Issue Type: Bug
> Components: Tests
> Affects Versions: 1.10.0
> Reporter: Congxian Qiu(klion26)
> Priority: Major
>
> When verifying the RC of release-1.10.1, found that
> `TaskExecutorSubmissionTest#testFailingScheduleOrUpdateConsumers` will fail
> because of Timeout sometime.
> I run this test locally in IDEA, found the following exception(locally in
> only encounter 2 in 1000 times)
> {code:java}
> java.lang.InterruptedExceptionjava.lang.InterruptedException at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
> at scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:212)
> at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:222) at
> scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:227) at
> scala.concurrent.Await$$anonfun$result$1.apply(package.scala:190) at
> scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53)
> at scala.concurrent.Await$.result(package.scala:190) at
> akka.event.LoggingBus$class.akka$event$LoggingBus$$addLogger(Logging.scala:182)
> at
> akka.event.LoggingBus$$anonfun$4$$anonfun$apply$4.apply(Logging.scala:117) at
> akka.event.LoggingBus$$anonfun$4$$anonfun$apply$4.apply(Logging.scala:116) at
> scala.util.Success$$anonfun$map$1.apply(Try.scala:237) at
> scala.util.Try$.apply(Try.scala:192) at scala.util.Success.map(Try.scala:237)
> at akka.event.LoggingBus$$anonfun$4.apply(Logging.scala:116) at
> akka.event.LoggingBus$$anonfun$4.apply(Logging.scala:113) at
> scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:683)
> at scala.collection.Iterator$class.foreach(Iterator.scala:891) at
> scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at
> scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at
> scala.collection.AbstractIterable.foreach(Iterable.scala:54) at
> scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:682) at
> akka.event.LoggingBus$class.startDefaultLoggers(Logging.scala:113) at
> akka.event.EventStream.startDefaultLoggers(EventStream.scala:22) at
> akka.actor.LocalActorRefProvider.init(ActorRefProvider.scala:662) at
> akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:874) at
> akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:870) at
> akka.actor.ActorSystemImpl._start(ActorSystem.scala:870) at
> akka.actor.ActorSystemImpl.start(ActorSystem.scala:891) at
> akka.actor.RobustActorSystem$.internalApply(RobustActorSystem.scala:96) at
> akka.actor.RobustActorSystem$.apply(RobustActorSystem.scala:70) at
> akka.actor.RobustActorSystem$.create(RobustActorSystem.scala:55) at
> org.apache.flink.runtime.akka.AkkaUtils$.createActorSystem(AkkaUtils.scala:125)
> at
> org.apache.flink.runtime.akka.AkkaUtils$.createActorSystem(AkkaUtils.scala:113)
> at
> org.apache.flink.runtime.akka.AkkaUtils$.createLocalActorSystem(AkkaUtils.scala:68)
> at
> org.apache.flink.runtime.akka.AkkaUtils.createLocalActorSystem(AkkaUtils.scala)
> at
> org.apache.flink.runtime.rpc.TestingRpcService.<init>(TestingRpcService.java:74)
> at
> org.apache.flink.runtime.rpc.TestingRpcService.<init>(TestingRpcService.java:67)
> at
> org.apache.flink.runtime.taskexecutor.TaskSubmissionTestEnvironment$Builder.build(TaskSubmissionTestEnvironment.java:349)
> at
> org.apache.flink.runtime.taskexecutor.TaskExecutorSubmissionTest.testFailingScheduleOrUpdateConsumers(TaskExecutorSubmissionTest.java:544)
> at sun.reflect.GeneratedMethodAccessor34.invoke(Unknown Source) at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498) at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
> at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
> at
> org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
> at
> org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298)
> at
> org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266) at
> java.lang.Thread.run(Thread.java:748)
> org.junit.runners.model.TestTimedOutException: test timed out after 10000
> milliseconds
> at sun.misc.Unsafe.park(Native Method) at
> java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
> at scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:212)
> at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:222) at
> scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:227) at
> scala.concurrent.Await$$anonfun$result$1.apply(package.scala:190) at
> scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53)
> at scala.concurrent.Await$.result(package.scala:190) at
> akka.event.LoggingBus$class.akka$event$LoggingBus$$addLogger(Logging.scala:182)
> at
> akka.event.LoggingBus$$anonfun$4$$anonfun$apply$4.apply(Logging.scala:117) at
> akka.event.LoggingBus$$anonfun$4$$anonfun$apply$4.apply(Logging.scala:116) at
> scala.util.Success$$anonfun$map$1.apply(Try.scala:237) at
> scala.util.Try$.apply(Try.scala:192) at scala.util.Success.map(Try.scala:237)
> at akka.event.LoggingBus$$anonfun$4.apply(Logging.scala:116) at
> akka.event.LoggingBus$$anonfun$4.apply(Logging.scala:113) at
> scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:683)
> at scala.collection.Iterator$class.foreach(Iterator.scala:891) at
> scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at
> scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at
> scala.collection.AbstractIterable.foreach(Iterable.scala:54) at
> scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:682) at
> akka.event.LoggingBus$class.startDefaultLoggers(Logging.scala:113) at
> akka.event.EventStream.startDefaultLoggers(EventStream.scala:22) at
> akka.actor.LocalActorRefProvider.init(ActorRefProvider.scala:662) at
> akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:874) at
> akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:870) at
> akka.actor.ActorSystemImpl._start(ActorSystem.scala:870) at
> akka.actor.ActorSystemImpl.start(ActorSystem.scala:891) at
> akka.actor.RobustActorSystem$.internalApply(RobustActorSystem.scala:96) at
> akka.actor.RobustActorSystem$.apply(RobustActorSystem.scala:70) at
> akka.actor.RobustActorSystem$.create(RobustActorSystem.scala:55) at
> org.apache.flink.runtime.akka.AkkaUtils$.createActorSystem(AkkaUtils.scala:125)
> at
> org.apache.flink.runtime.akka.AkkaUtils$.createActorSystem(AkkaUtils.scala:113)
> at
> org.apache.flink.runtime.akka.AkkaUtils$.createLocalActorSystem(AkkaUtils.scala:68)
> at
> org.apache.flink.runtime.akka.AkkaUtils.createLocalActorSystem(AkkaUtils.scala)
> at
> org.apache.flink.runtime.rpc.TestingRpcService.<init>(TestingRpcService.java:74)
> at
> org.apache.flink.runtime.rpc.TestingRpcService.<init>(TestingRpcService.java:67)
> at
> org.apache.flink.runtime.taskexecutor.TaskSubmissionTestEnvironment$Builder.build(TaskSubmissionTestEnvironment.java:349)
> at
> org.apache.flink.runtime.taskexecutor.TaskExecutorSubmissionTest.testFailingScheduleOrUpdateConsumers(TaskExecutorSubmissionTest.java:544)
> at sun.reflect.GeneratedMethodAccessor34.invoke(Unknown Source) at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498) at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
> at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
> at
> org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
> at
> org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298)
> at
> org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266) at
> java.lang.Thread.run(Thread.java:748)
> {code}
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
