[
https://issues.apache.org/jira/browse/FLINK-15388?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17003151#comment-17003151
]
Xintong Song commented on FLINK-15388:
--------------------------------------
It seems there was some network problem at the time of this failure.
I found the following error message in your taskmanager log.
{code}
2019-12-25 07:52:35.957 [flink-scheduler-1] WARN
org.apache.flink.runtime.taskexecutor.TaskExecutor - Could not send heartbeat
to target with id 2981310fa47a49b417e51ed799ff77d4.
java.util.concurrent.CompletionException: akka.pattern.AskTimeoutException: Ask
timed out on [Actor[akka://flink/user/taskmanager_0#1008850236]] after [10000
ms]. Sender[null] sent message of type
"org.apache.flink.runtime.rpc.messages.CallAsync".
at
java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292)
~[na:1.8.0_211]
at
java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308)
~[na:1.8.0_211]
at
java.util.concurrent.CompletableFuture.uniAccept(CompletableFuture.java:647)
~[na:1.8.0_211]
at
java.util.concurrent.CompletableFuture$UniAccept.tryFire(CompletableFuture.java:632)
~[na:1.8.0_211]
at
java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)
~[na:1.8.0_211]
at
java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1977)
~[na:1.8.0_211]
at
org.apache.flink.runtime.concurrent.FutureUtils$1.onComplete(FutureUtils.java:815)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at akka.dispatch.OnComplete.internal(Future.scala:258)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at akka.dispatch.OnComplete.internal(Future.scala:256)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at akka.dispatch.japi$CallbackBridge.apply(Future.scala:186)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at akka.dispatch.japi$CallbackBridge.apply(Future.scala:183)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:36)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
org.apache.flink.runtime.concurrent.Executors$DirectExecutionContext.execute(Executors.java:74)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:44)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:252)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
akka.pattern.PromiseActorRef$$anonfun$1.apply$mcV$sp(AskSupport.scala:603)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at akka.actor.Scheduler$$anon$4.run(Scheduler.scala:126)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
scala.concurrent.Future$InternalCallbackExecutor$.unbatchedExecute(Future.scala:601)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
scala.concurrent.BatchingExecutor$class.execute(BatchingExecutor.scala:109)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
scala.concurrent.Future$InternalCallbackExecutor$.execute(Future.scala:599)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
akka.actor.LightArrayRevolverScheduler$TaskHolder.executeTask(LightArrayRevolverScheduler.scala:329)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
akka.actor.LightArrayRevolverScheduler$$anon$4.executeBucket$1(LightArrayRevolverScheduler.scala:280)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
akka.actor.LightArrayRevolverScheduler$$anon$4.nextTick(LightArrayRevolverScheduler.scala:284)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
akka.actor.LightArrayRevolverScheduler$$anon$4.run(LightArrayRevolverScheduler.scala:236)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_211]
Caused by: akka.pattern.AskTimeoutException: Ask timed out on
[Actor[akka://flink/user/taskmanager_0#1008850236]] after [10000 ms].
Sender[null] sent message of type
"org.apache.flink.runtime.rpc.messages.CallAsync".
at
akka.pattern.PromiseActorRef$$anonfun$1.apply$mcV$sp(AskSupport.scala:604)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
... 9 common frames omitted
{code}
And in the jobmanager log about the same time range, the following error
message with the same taskmanager id.
{code}
2019-12-25 07:51:48.640 [flink-akka.actor.default-dispatcher-24] WARN
akka.remote.ReliableDeliverySupervisor
flink-akka.remote.default-remote-dispatcher-17 - Association with remote system
[akka.tcp://[email protected]:27065] has failed, address is now gated for [50]
ms. Reason: [Association failed with [akka.tcp://[email protected]:27065]]
Caused by: [Connection refused: /10.1.209.158:27065]
2019-12-25 07:51:55.743 [flink-scheduler-1] ERROR
o.a.f.r.r.handler.taskmanager.TaskManagerStdoutFileHandler - Failed to
transfer file from TaskExecutor bae00218c818157649eb9e3c533b86af.
java.util.concurrent.CompletionException: akka.pattern.AskTimeoutException: Ask
timed out on
[Actor[akka.tcp://[email protected]:47375/user/resourcemanager#-682605099]]
after [10000 ms]. Sender[null] sent message of type
"org.apache.flink.runtime.rpc.messages.RemoteFencedMessage".
at
java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292)
~[na:1.8.0_131]
at
java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308)
~[na:1.8.0_131]
at
java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:593)
~[na:1.8.0_131]
at
java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577)
~[na:1.8.0_131]
at
java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)
~[na:1.8.0_131]
at
java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1977)
~[na:1.8.0_131]
at
org.apache.flink.runtime.concurrent.FutureUtils$1.onComplete(FutureUtils.java:815)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at akka.dispatch.OnComplete.internal(Future.scala:258)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at akka.dispatch.OnComplete.internal(Future.scala:256)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at akka.dispatch.japi$CallbackBridge.apply(Future.scala:186)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at akka.dispatch.japi$CallbackBridge.apply(Future.scala:183)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:36)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
org.apache.flink.runtime.concurrent.Executors$DirectExecutionContext.execute(Executors.java:74)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:44)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:252)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
akka.pattern.PromiseActorRef$$anonfun$1.apply$mcV$sp(AskSupport.scala:603)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at akka.actor.Scheduler$$anon$4.run(Scheduler.scala:126)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
scala.concurrent.Future$InternalCallbackExecutor$.unbatchedExecute(Future.scala:601)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
scala.concurrent.BatchingExecutor$class.execute(BatchingExecutor.scala:109)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
scala.concurrent.Future$InternalCallbackExecutor$.execute(Future.scala:599)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
akka.actor.LightArrayRevolverScheduler$TaskHolder.executeTask(LightArrayRevolverScheduler.scala:329)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
akka.actor.LightArrayRevolverScheduler$$anon$4.executeBucket$1(LightArrayRevolverScheduler.scala:280)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
akka.actor.LightArrayRevolverScheduler$$anon$4.nextTick(LightArrayRevolverScheduler.scala:284)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at
akka.actor.LightArrayRevolverScheduler$$anon$4.run(LightArrayRevolverScheduler.scala:236)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_131]
Caused by: akka.pattern.AskTimeoutException: Ask timed out on
[Actor[akka.tcp://[email protected]:47375/user/resourcemanager#-682605099]]
after [10000 ms]. Sender[null] sent message of type
"org.apache.flink.runtime.rpc.messages.RemoteFencedMessage".
at
akka.pattern.PromiseActorRef$$anonfun$1.apply$mcV$sp(AskSupport.scala:604)
~[flink-dist_2.11-1.8.0.jar:1.8.0]
... 9 common frames omitted
2019-12-25 07:51:55.743 [flink-scheduler-1] ERROR
o.a.f.r.r.handler.taskmanager.TaskManagerStdoutFileHandler - Failed to
transfer file from TaskExecutor bae00218c818157649eb9e3c533b86af.
{code}
> The assigned slot bae00218c818157649eb9e3c533b86af_32 was removed.
> ------------------------------------------------------------------
>
> Key: FLINK-15388
> URL: https://issues.apache.org/jira/browse/FLINK-15388
> Project: Flink
> Issue Type: Bug
> Components: Runtime / Task
> Affects Versions: 1.8.0
> Environment: model : standalone,not yarn
> version : flink 1.8.0
> configration :
> jobmanager.heap.size: 4096m
> taskmanager.heap.size: 144gb
> taskmanager.numberOfTaskSlots: 48
> taskmanager.memory.fraction: 0.7
> taskmanager.memory.off-heap: false
> parallelism.default: 1
>
> Reporter: hiliuxg
> Priority: Major
>
> the taskmanager's slot was removed , there was not full gc or oom , what's
> the problem ? the error bellow
> {code:java}
> org.apache.flink.util.FlinkException: The assigned slot
> bae00218c818157649eb9e3c533b86af_32 was removed.
> at
> org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager.removeSlot(SlotManager.java:893)
> at
> org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager.removeSlots(SlotManager.java:863)
> at
> org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager.internalUnregisterTaskManager(SlotManager.java:1058)
> at
> org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager.unregisterTaskManager(SlotManager.java:385)
> at
> org.apache.flink.runtime.resourcemanager.ResourceManager.closeTaskManagerConnection(ResourceManager.java:847)
> at
> org.apache.flink.runtime.resourcemanager.ResourceManager$TaskManagerHeartbeatListener$1.run(ResourceManager.java:1161)
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(AkkaRpcActor.java:392)
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:185)
> at
> org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:74)
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.onReceive(AkkaRpcActor.java:147)
> at
> org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.onReceive(FencedAkkaRpcActor.java:40)
> at
> akka.actor.UntypedActor$$anonfun$receive$1.applyOrElse(UntypedActor.scala:165)
> at akka.actor.Actor$class.aroundReceive(Actor.scala:502)
> at akka.actor.UntypedActor.aroundReceive(UntypedActor.scala:95)
> at akka.actor.ActorCell.receiveMessage(ActorCell.scala:526)
> at akka.actor.ActorCell.invoke(ActorCell.scala:495)
> at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:257)
> at akka.dispatch.Mailbox.run(Mailbox.scala:224)
> at akka.dispatch.Mailbox.exec(Mailbox.scala:234)
> at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
> at
> scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
> at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
