I am running Flink 1.7 on K8S. I am not sure how to debug this issue. I turned on debug on JM/TM.
I am not sure this part is related or not. How could an Actor suddenly disappear? ===== 2018-11-09 04:47:19,480 DEBUG org.apache.flink.runtime.rest.handler.legacy.metrics.MetricFetcher - Query metrics for akka://flink-metrics/user/MetricQueryService. 2018-11-09 04:47:19,577 DEBUG org.apache.flink.runtime.rest.handler.legacy.metrics.MetricFetcher - Retrieve metric query service gateway for akka.tcp://flink-metrics@fps-flink-taskmanager-6f8f687fc8-b5qmm :34429/user/MetricQueryService_0cfaa7b8f193a8002f121282298c58ac 2018-11-09 04:47:19,577 DEBUG org.apache.flink.runtime.rest.handler.legacy.metrics.MetricFetcher - Retrieve metric query service gateway for akka.tcp://flink-metrics@fps-flink-taskmanager-6f8f687fc8-ckwzp :43062/user/MetricQueryService_1930889e5b6c51cbe57428f9a664e4dc 2018-11-09 04:47:19,578 DEBUG org.apache.flink.runtime.rest.handler.legacy.metrics.MetricFetcher - Retrieve metric query service gateway for akka.tcp://flink-metrics@fps-flink-taskmanager-6f8f687fc8-bzq2w :39393/user/MetricQueryService_fcaf62301df0a6aeb29a65470cfe1e7a 2018-11-09 04:47:19,585 TRACE org.apache.flink.runtime.rest.FileUploadHandler - Received request. URL:/jobs/00000000000000000000000000000000 Method:GET 2018-11-09 04:47:19,613 DEBUG org.apache.flink.runtime.rest.handler.legacy.metrics.MetricFetcher - Could not retrieve QueryServiceGateway. java.util.concurrent.CompletionException: akka.actor.ActorNotFound: Actor not found for: ActorSelection[Anchor(akka.tcp://flink-metrics@fps-flink-taskmanager-6f8f687fc8-ckwzp:43062/), Path(/user/MetricQueryService_1930889e5b6c51cbe57428f9a664e4dc)] at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292) at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308) at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:593) at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577) at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1977) at org.apache.flink.runtime.concurrent.FutureUtils$1.onComplete(FutureUtils.java:772) at akka.dispatch.OnComplete.internal(Future.scala:258) at akka.dispatch.OnComplete.internal(Future.scala:256) at akka.dispatch.japi$CallbackBridge.apply(Future.scala:186) at akka.dispatch.japi$CallbackBridge.apply(Future.scala:183) at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60) at org.apache.flink.runtime.concurrent.Executors$DirectExecutionContext.execute(Executors.java:83) at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:68) at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1(Promise.scala:284) at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1$adapted(Promise.scala:284) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:284) at scala.concurrent.Promise.complete(Promise.scala:49) at scala.concurrent.Promise.complete$(Promise.scala:48) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:183) at scala.concurrent.Promise.failure(Promise.scala:100) at scala.concurrent.Promise.failure$(Promise.scala:100) at scala.concurrent.impl.Promise$DefaultPromise.failure(Promise.scala:183) at akka.actor.ActorSelection.$anonfun$resolveOne$1(ActorSelection.scala:68) at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60) at akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:55) at akka.dispatch.BatchingExecutor$Batch.run(BatchingExecutor.scala:73) at akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.unbatchedExecute(Future.scala:76) at akka.dispatch.BatchingExecutor.execute(BatchingExecutor.scala:120) at akka.dispatch.BatchingExecutor.execute$(BatchingExecutor.scala:114) at akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.execute(Future.scala:75) at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:68) at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1(Promise.scala:284) at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1$adapted(Promise.scala:284) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:284) at akka.pattern.PromiseActorRef.$bang(AskSupport.scala:538) at akka.actor.EmptyLocalActorRef.specialHandle(ActorRef.scala:558) at akka.actor.DeadLetterActorRef.specialHandle(ActorRef.scala:595) at akka.actor.DeadLetterActorRef.$bang(ActorRef.scala:584) at akka.remote.RemoteActorRefProvider$RemoteDeadLetterActorRef.$bang(RemoteActorRefProvider.scala:98) at akka.remote.EndpointWriter.postStop(Endpoint.scala:593) at akka.actor.Actor.aroundPostStop(Actor.scala:515) at akka.actor.Actor.aroundPostStop$(Actor.scala:515) at akka.remote.EndpointActor.aroundPostStop(Endpoint.scala:446) at akka.actor.dungeon.FaultHandling.finishTerminate(FaultHandling.scala:210) at akka.actor.dungeon.FaultHandling.terminate(FaultHandling.scala:172) at akka.actor.dungeon.FaultHandling.terminate$(FaultHandling.scala:142) at akka.actor.ActorCell.terminate(ActorCell.scala:374) at akka.actor.ActorCell.invokeAll$1(ActorCell.scala:467) at akka.actor.ActorCell.systemInvoke(ActorCell.scala:483) at akka.dispatch.Mailbox.processAllSystemMessages(Mailbox.scala:282) at akka.dispatch.Mailbox.run(Mailbox.scala:223) at akka.dispatch.Mailbox.exec(Mailbox.scala:234) at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:157) Caused by: akka.actor.ActorNotFound: Actor not found for: ActorSelection[Anchor(akka.tcp://flink-metrics@fps-flink-taskmanager-6f8f687fc8-ckwzp:43062/), Path(/user/MetricQueryService_1930889e5b6c51cbe57428f9a664e4dc)] ... 34 more 2018-11-09 04:47:19,680 DEBUG org.apache.flink.runtime.rest.handler.legacy.metrics.MetricFetcher - Could not retrieve QueryServiceGateway. java.util.concurrent.CompletionException: akka.actor.ActorNotFound: Actor not found for: ActorSelection[Anchor(akka.tcp://flink-metrics@fps-flink-taskmanager-6f8f687fc8-bzq2w:39393/), Path(/user/MetricQueryService_fcaf62301df0a6aeb29a65470cfe1e7a)] at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292) at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308) at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:593) at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577) at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1977) at org.apache.flink.runtime.concurrent.FutureUtils$1.onComplete(FutureUtils.java:772) at akka.dispatch.OnComplete.internal(Future.scala:258) at akka.dispatch.OnComplete.internal(Future.scala:256) at akka.dispatch.japi$CallbackBridge.apply(Future.scala:186) at akka.dispatch.japi$CallbackBridge.apply(Future.scala:183) at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60) at org.apache.flink.runtime.concurrent.Executors$DirectExecutionContext.execute(Executors.java:83) at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:68) at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1(Promise.scala:284) at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1$adapted(Promise.scala:284) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:284) at scala.concurrent.Promise.complete(Promise.scala:49) at scala.concurrent.Promise.complete$(Promise.scala:48) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:183) at scala.concurrent.Promise.failure(Promise.scala:100) at scala.concurrent.Promise.failure$(Promise.scala:100) at scala.concurrent.impl.Promise$DefaultPromise.failure(Promise.scala:183) at akka.actor.ActorSelection.$anonfun$resolveOne$1(ActorSelection.scala:68) at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60) at akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:55) at akka.dispatch.BatchingExecutor$Batch.run(BatchingExecutor.scala:73) at akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.unbatchedExecute(Future.scala:76) at akka.dispatch.BatchingExecutor.execute(BatchingExecutor.scala:120) at akka.dispatch.BatchingExecutor.execute$(BatchingExecutor.scala:114) at akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.execute(Future.scala:75) at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:68) at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1(Promise.scala:284) at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1$adapted(Promise.scala:284) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:284) at akka.pattern.PromiseActorRef.$bang(AskSupport.scala:538) at akka.actor.EmptyLocalActorRef.specialHandle(ActorRef.scala:558) at akka.actor.DeadLetterActorRef.specialHandle(ActorRef.scala:595) at akka.actor.DeadLetterActorRef.$bang(ActorRef.scala:584) at akka.remote.RemoteActorRefProvider$RemoteDeadLetterActorRef.$bang(RemoteActorRefProvider.scala:98) at akka.remote.EndpointWriter.postStop(Endpoint.scala:593) at akka.actor.Actor.aroundPostStop(Actor.scala:515) at akka.actor.Actor.aroundPostStop$(Actor.scala:515) at akka.remote.EndpointActor.aroundPostStop(Endpoint.scala:446) at akka.actor.dungeon.FaultHandling.finishTerminate(FaultHandling.scala:210) at akka.actor.dungeon.FaultHandling.terminate(FaultHandling.scala:172) at akka.actor.dungeon.FaultHandling.terminate$(FaultHandling.scala:142) at akka.actor.ActorCell.terminate(ActorCell.scala:374) at akka.actor.ActorCell.invokeAll$1(ActorCell.scala:467) at akka.actor.ActorCell.systemInvoke(ActorCell.scala:483) at akka.dispatch.Mailbox.processAllSystemMessages(Mailbox.scala:282) at akka.dispatch.Mailbox.run(Mailbox.scala:223) at akka.dispatch.Mailbox.exec(Mailbox.scala:234) at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:157) Caused by: akka.actor.ActorNotFound: Actor not found for: ActorSelection[Anchor(akka.tcp://flink-metrics@fps-flink-taskmanager-6f8f687fc8-bzq2w:39393/), Path(/user/MetricQueryService_fcaf62301df0a6aeb29a65470cfe1e7a)] ... 34 more 2018-11-09 04:47:19,682 DEBUG org.apache.flink.runtime.rest.handler.legacy.metrics.MetricFetcher - Could not retrieve QueryServiceGateway. java.util.concurrent.CompletionException: akka.actor.ActorNotFound: Actor not found for: ActorSelection[Anchor(akka.tcp://flink-metrics@fps-flink-taskmanager-6f8f687fc8-b5qmm:34429/), Path(/user/MetricQueryService_0cfaa7b8f193a8002f121282298c58ac)] at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292) at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308) at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:593) at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577) at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1977) at org.apache.flink.runtime.concurrent.FutureUtils$1.onComplete(FutureUtils.java:772) at akka.dispatch.OnComplete.internal(Future.scala:258) at akka.dispatch.OnComplete.internal(Future.scala:256) at akka.dispatch.japi$CallbackBridge.apply(Future.scala:186) at akka.dispatch.japi$CallbackBridge.apply(Future.scala:183) at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60) at org.apache.flink.runtime.concurrent.Executors$DirectExecutionContext.execute(Executors.java:83) at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:68) at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1(Promise.scala:284) at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1$adapted(Promise.scala:284) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:284) at scala.concurrent.Promise.complete(Promise.scala:49) at scala.concurrent.Promise.complete$(Promise.scala:48) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:183) at scala.concurrent.Promise.failure(Promise.scala:100) at scala.concurrent.Promise.failure$(Promise.scala:100) at scala.concurrent.impl.Promise$DefaultPromise.failure(Promise.scala:183) at akka.actor.ActorSelection.$anonfun$resolveOne$1(ActorSelection.scala:68) at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60) at akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:55) at akka.dispatch.BatchingExecutor$Batch.run(BatchingExecutor.scala:73) at akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.unbatchedExecute(Future.scala:76) at akka.dispatch.BatchingExecutor.execute(BatchingExecutor.scala:120) at akka.dispatch.BatchingExecutor.execute$(BatchingExecutor.scala:114) at akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.execute(Future.scala:75) at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:68) at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1(Promise.scala:284) at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1$adapted(Promise.scala:284) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:284) at akka.pattern.PromiseActorRef.$bang(AskSupport.scala:538) at akka.actor.EmptyLocalActorRef.specialHandle(ActorRef.scala:558) at akka.actor.DeadLetterActorRef.specialHandle(ActorRef.scala:595) at akka.actor.DeadLetterActorRef.$bang(ActorRef.scala:584) at akka.remote.RemoteActorRefProvider$RemoteDeadLetterActorRef.$bang(RemoteActorRefProvider.scala:98) at akka.remote.EndpointWriter.postStop(Endpoint.scala:593) at akka.actor.Actor.aroundPostStop(Actor.scala:515) at akka.actor.Actor.aroundPostStop$(Actor.scala:515) at akka.remote.EndpointActor.aroundPostStop(Endpoint.scala:446) at akka.actor.dungeon.FaultHandling.finishTerminate(FaultHandling.scala:210) at akka.actor.dungeon.FaultHandling.terminate(FaultHandling.scala:172) at akka.actor.dungeon.FaultHandling.terminate$(FaultHandling.scala:142) at akka.actor.ActorCell.terminate(ActorCell.scala:374) at akka.actor.ActorCell.invokeAll$1(ActorCell.scala:467) at akka.actor.ActorCell.systemInvoke(ActorCell.scala:483) at akka.dispatch.Mailbox.processAllSystemMessages(Mailbox.scala:282) at akka.dispatch.Mailbox.run(Mailbox.scala:223) at akka.dispatch.Mailbox.exec(Mailbox.scala:234) at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:157) Caused by: akka.actor.ActorNotFound: Actor not found for: ActorSelection[Anchor(akka.tcp://flink-metrics@fps-flink-taskmanager-6f8f687fc8-b5qmm:34429/), Path(/user/MetricQueryService_0cfaa7b8f193a8002f121282298c58ac)] ... 34 more 2018-11-09 04:47:19,682 TRACE org.apache.flink.runtime.rest.handler.job.JobDetailsHandler - Received request /jobs/00000000000000000000000000000000. 2018-11-09 04:47:19,683 TRACE org.apache.flink.runtime.rest.handler.job.JobDetailsHandler - Starting request processing. 2018-11-09 04:47:19,777 TRACE org.apache.flink.runtime.rest.FileUploadHandler - Received request. URL:/jobs/overview Method:GET 2018-11-09 04:47:19,779 TRACE org.apache.flink.runtime.rest.handler.job.JobsOverviewHandler - Received request /jobs/overview. 2018-11-09 04:47:19,779 TRACE org.apache.flink.runtime.rest.handler.job.JobsOverviewHandler - Starting request processing. 2018-11-09 04:47:22,687 TRACE org.apache.flink.runtime.rest.FileUploadHandler - Received request. URL:/jobs/00000000000000000000000000000000/config Method:GET =======
