[
https://issues.apache.org/jira/browse/FLINK-27295?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Yun Tang updated FLINK-27295:
-----------------------------
Description:
{code:java}
09:17:42,931 [flink-akka.actor.default-dispatcher-9] INFO
org.apache.flink.runtime.jobmaster.JobMaster [] - Trying to
recover from a global failure.
java.lang.IllegalStateException: Cannot mark for checkpoint 17, already marked
for checkpoint 16
at
org.apache.flink.runtime.operators.coordination.OperatorEventValve.markForCheckpoint(OperatorEventValve.java:113)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.checkpointCoordinatorInternal(OperatorCoordinatorHolder.java:302)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.lambda$checkpointCoordinator$0(OperatorCoordinatorHolder.java:230)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRunAsync$4(AkkaRpcActor.java:444)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(AkkaRpcActor.java:444)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:214)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:164)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at scala.PartialFunction.applyOrElse(PartialFunction.scala:123)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.actor.Actor.aroundReceive(Actor.scala:537)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.actor.Actor.aroundReceive$(Actor.scala:535)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.actor.ActorCell.invoke(ActorCell.scala:548)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.dispatch.Mailbox.run(Mailbox.scala:231)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.dispatch.Mailbox.exec(Mailbox.scala:243)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
[?:1.8.0_292]
at
java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
[?:1.8.0_292]
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
[?:1.8.0_292]
at
java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
[?:1.8.0_292]
09:17:42,932 [flink-akka.actor.default-dispatcher-9] INFO
org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Job Flink
Streaming Job (3bb746b6088ca587b3664599a7bae6e7) switched from state RUNNING to
FAILING.
org.apache.flink.runtime.JobException: Recovery is suppressed by
FixedDelayRestartBackoffTimeStrategy(maxNumberRestartAttempts=5,
backoffTimeMS=100)
at
org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.handleFailure(ExecutionFailureHandler.java:138)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.getGlobalFailureHandlingResult(ExecutionFailureHandler.java:101)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.scheduler.DefaultScheduler.handleGlobalFailure(DefaultScheduler.java:322)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.checkpointCoordinatorInternal(OperatorCoordinatorHolder.java:307)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.lambda$checkpointCoordinator$0(OperatorCoordinatorHolder.java:230)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRunAsync$4(AkkaRpcActor.java:444)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(AkkaRpcActor.java:444)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:214)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:164)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at scala.PartialFunction.applyOrElse(PartialFunction.scala:123)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.actor.Actor.aroundReceive(Actor.scala:537)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.actor.Actor.aroundReceive$(Actor.scala:535)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.actor.ActorCell.invoke(ActorCell.scala:548)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.dispatch.Mailbox.run(Mailbox.scala:231)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at akka.dispatch.Mailbox.exec(Mailbox.scala:243)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
[?:1.8.0_292]
at
java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
[?:1.8.0_292]
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
[?:1.8.0_292]
at
java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
[?:1.8.0_292]
Caused by: java.lang.IllegalStateException: Cannot mark for checkpoint 17,
already marked for checkpoint 16
at
org.apache.flink.runtime.operators.coordination.OperatorEventValve.markForCheckpoint(OperatorEventValve.java:113)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.checkpointCoordinatorInternal(OperatorCoordinatorHolder.java:302)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
... 27 more
09:17:42,930 [ Checkpoint Timer] WARN
org.apache.flink.runtime.checkpoint.CheckpointFailureManager [] - Failed to
trigger or complete checkpoint 17 for job 3bb746b6088ca587b3664599a7bae6e7. (0
consecutive failed attempts so far)
org.apache.flink.runtime.checkpoint.CheckpointException: Trigger checkpoint
failure.
at
org.apache.flink.runtime.checkpoint.CheckpointCoordinator.lambda$getCheckpointException$19(CheckpointCoordinator.java:2167)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at java.util.Optional.orElseGet(Optional.java:267) ~[?:1.8.0_292]
at
org.apache.flink.runtime.checkpoint.CheckpointCoordinator.getCheckpointException(CheckpointCoordinator.java:2166)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.checkpoint.CheckpointCoordinator.onTriggerFailure(CheckpointCoordinator.java:938)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.checkpoint.CheckpointCoordinator.onTriggerFailure(CheckpointCoordinator.java:916)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.checkpoint.CheckpointCoordinator.lambda$startTriggeringCheckpoint$7(CheckpointCoordinator.java:642)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:836)
[?:1.8.0_292]
at
java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:811)
[?:1.8.0_292]
at
java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:456)
[?:1.8.0_292]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[?:1.8.0_292]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[?:1.8.0_292]
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
[?:1.8.0_292]
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
[?:1.8.0_292]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[?:1.8.0_292]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[?:1.8.0_292]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_292]
Caused by: java.lang.IllegalStateException: Cannot mark for checkpoint 17,
already marked for checkpoint 16
at
org.apache.flink.runtime.operators.coordination.OperatorEventValve.markForCheckpoint(OperatorEventValve.java:113)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.checkpointCoordinatorInternal(OperatorCoordinatorHolder.java:302)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.lambda$checkpointCoordinator$0(OperatorCoordinatorHolder.java:230)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRunAsync$4(AkkaRpcActor.java:444)
~[?:?]
at
org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68)
~[?:?]
at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(AkkaRpcActor.java:444)
~[?:?]
at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:214)
~[?:?]
at
org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78)
~[?:?]
at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:164)
~[?:?]
at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24) ~[?:?]
at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20) ~[?:?]
at scala.PartialFunction.applyOrElse(PartialFunction.scala:123)
~[scala-library-2.12.7.jar:?]
at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122)
~[scala-library-2.12.7.jar:?]
at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20)
~[?:?]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
~[scala-library-2.12.7.jar:?]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
~[scala-library-2.12.7.jar:?]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
~[scala-library-2.12.7.jar:?]
at akka.actor.Actor.aroundReceive(Actor.scala:537) ~[?:?]
at akka.actor.Actor.aroundReceive$(Actor.scala:535) ~[?:?]
at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220)
~[?:?]
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580) ~[?:?]
at akka.actor.ActorCell.invoke(ActorCell.scala:548) ~[?:?]
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270) ~[?:?]
at akka.dispatch.Mailbox.run(Mailbox.scala:231) ~[?:?]
at akka.dispatch.Mailbox.exec(Mailbox.scala:243) ~[?:?]
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
~[?:1.8.0_292]
at
java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
~[?:1.8.0_292]
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
~[?:1.8.0_292]
at
java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
~[?:1.8.0_292]
{code}
Instace:
[https://dev.azure.com/apache-flink/apache-flink/_build/results?buildId=34771&view=logs&j=5c8e7682-d68f-54d1-16a2-a09310218a49&t=86f654fa-ab48-5c1a-25f4-7e7f6afb9bba]
cc [~pnowojski] , [~becket_qin]
was:
{code:java}
09:17:42,931 [flink-akka.actor.default-dispatcher-9] INFO
org.apache.flink.runtime.jobmaster.JobMaster [] - Trying to
recover from a global failure.java.lang.IllegalStateException: Cannot mark for
checkpoint 17, already marked for checkpoint 16 at
org.apache.flink.runtime.operators.coordination.OperatorEventValve.markForCheckpoint(OperatorEventValve.java:113)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.checkpointCoordinatorInternal(OperatorCoordinatorHolder.java:302)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.lambda$checkpointCoordinator$0(OperatorCoordinatorHolder.java:230)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRunAsync$4(AkkaRpcActor.java:444)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(AkkaRpcActor.java:444)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:214)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:164)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
scala.PartialFunction.applyOrElse(PartialFunction.scala:123)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
scala.PartialFunction.applyOrElse$(PartialFunction.scala:122)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.actor.Actor.aroundReceive(Actor.scala:537)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.actor.Actor.aroundReceive$(Actor.scala:535)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.actor.ActorCell.receiveMessage(ActorCell.scala:580)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.actor.ActorCell.invoke(ActorCell.scala:548)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.dispatch.Mailbox.run(Mailbox.scala:231)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.dispatch.Mailbox.exec(Mailbox.scala:243)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) [?:1.8.0_292]
at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
[?:1.8.0_292] at
java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
[?:1.8.0_292] at
java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
[?:1.8.0_292]09:17:42,932 [flink-akka.actor.default-dispatcher-9] INFO
org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Job Flink
Streaming Job (3bb746b6088ca587b3664599a7bae6e7) switched from state RUNNING to
FAILING.org.apache.flink.runtime.JobException: Recovery is suppressed by
FixedDelayRestartBackoffTimeStrategy(maxNumberRestartAttempts=5,
backoffTimeMS=100) at
org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.handleFailure(ExecutionFailureHandler.java:138)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.getGlobalFailureHandlingResult(ExecutionFailureHandler.java:101)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.scheduler.DefaultScheduler.handleGlobalFailure(DefaultScheduler.java:322)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.checkpointCoordinatorInternal(OperatorCoordinatorHolder.java:307)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.lambda$checkpointCoordinator$0(OperatorCoordinatorHolder.java:230)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRunAsync$4(AkkaRpcActor.java:444)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(AkkaRpcActor.java:444)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:214)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:164)
~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
scala.PartialFunction.applyOrElse(PartialFunction.scala:123)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
scala.PartialFunction.applyOrElse$(PartialFunction.scala:122)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.actor.Actor.aroundReceive(Actor.scala:537)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.actor.Actor.aroundReceive$(Actor.scala:535)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.actor.ActorCell.receiveMessage(ActorCell.scala:580)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.actor.ActorCell.invoke(ActorCell.scala:548)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.dispatch.Mailbox.run(Mailbox.scala:231)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
akka.dispatch.Mailbox.exec(Mailbox.scala:243)
[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at
java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) [?:1.8.0_292]
at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
[?:1.8.0_292] at
java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
[?:1.8.0_292] at
java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
[?:1.8.0_292]Caused by: java.lang.IllegalStateException: Cannot mark for
checkpoint 17, already marked for checkpoint 16 at
org.apache.flink.runtime.operators.coordination.OperatorEventValve.markForCheckpoint(OperatorEventValve.java:113)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.checkpointCoordinatorInternal(OperatorCoordinatorHolder.java:302)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] ... 27 more09:17:42,930 [
Checkpoint Timer] WARN
org.apache.flink.runtime.checkpoint.CheckpointFailureManager [] - Failed to
trigger or complete checkpoint 17 for job 3bb746b6088ca587b3664599a7bae6e7. (0
consecutive failed attempts so
far)org.apache.flink.runtime.checkpoint.CheckpointException: Trigger checkpoint
failure. at
org.apache.flink.runtime.checkpoint.CheckpointCoordinator.lambda$getCheckpointException$19(CheckpointCoordinator.java:2167)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at
java.util.Optional.orElseGet(Optional.java:267) ~[?:1.8.0_292] at
org.apache.flink.runtime.checkpoint.CheckpointCoordinator.getCheckpointException(CheckpointCoordinator.java:2166)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.checkpoint.CheckpointCoordinator.onTriggerFailure(CheckpointCoordinator.java:938)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.checkpoint.CheckpointCoordinator.onTriggerFailure(CheckpointCoordinator.java:916)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.checkpoint.CheckpointCoordinator.lambda$startTriggeringCheckpoint$7(CheckpointCoordinator.java:642)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at
java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:836)
[?:1.8.0_292] at
java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:811)
[?:1.8.0_292] at
java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:456)
[?:1.8.0_292] at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[?:1.8.0_292] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[?:1.8.0_292] at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
[?:1.8.0_292] at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
[?:1.8.0_292] at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[?:1.8.0_292] at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[?:1.8.0_292] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_292]Caused
by: java.lang.IllegalStateException: Cannot mark for checkpoint 17, already
marked for checkpoint 16 at
org.apache.flink.runtime.operators.coordination.OperatorEventValve.markForCheckpoint(OperatorEventValve.java:113)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.checkpointCoordinatorInternal(OperatorCoordinatorHolder.java:302)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.lambda$checkpointCoordinator$0(OperatorCoordinatorHolder.java:230)
~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRunAsync$4(AkkaRpcActor.java:444)
~[?:?] at
org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68)
~[?:?] at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(AkkaRpcActor.java:444)
~[?:?] at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:214)
~[?:?] at
org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78)
~[?:?] at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:164)
~[?:?] at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24)
~[?:?] at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20)
~[?:?] at scala.PartialFunction.applyOrElse(PartialFunction.scala:123)
~[scala-library-2.12.7.jar:?] at
scala.PartialFunction.applyOrElse$(PartialFunction.scala:122)
~[scala-library-2.12.7.jar:?] at
akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20) ~[?:?]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
~[scala-library-2.12.7.jar:?] at
scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
~[scala-library-2.12.7.jar:?] at
scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
~[scala-library-2.12.7.jar:?] at
akka.actor.Actor.aroundReceive(Actor.scala:537) ~[?:?] at
akka.actor.Actor.aroundReceive$(Actor.scala:535) ~[?:?] at
akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220) ~[?:?] at
akka.actor.ActorCell.receiveMessage(ActorCell.scala:580) ~[?:?] at
akka.actor.ActorCell.invoke(ActorCell.scala:548) ~[?:?] at
akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270) ~[?:?] at
akka.dispatch.Mailbox.run(Mailbox.scala:231) ~[?:?] at
akka.dispatch.Mailbox.exec(Mailbox.scala:243) ~[?:?] at
java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) ~[?:1.8.0_292]
at
java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
~[?:1.8.0_292] at
java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
~[?:1.8.0_292] at
java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
~[?:1.8.0_292] {code}
Instace:
[https://dev.azure.com/apache-flink/apache-flink/_build/results?buildId=34771&view=logs&j=5c8e7682-d68f-54d1-16a2-a09310218a49&t=86f654fa-ab48-5c1a-25f4-7e7f6afb9bba]
cc [~pnowojski] , [~becket_qin]
> UnalignedCheckpointITCase failed due to OperatorCoordinatorHolder cannot mark
> checkpoint
> ----------------------------------------------------------------------------------------
>
> Key: FLINK-27295
> URL: https://issues.apache.org/jira/browse/FLINK-27295
> Project: Flink
> Issue Type: Bug
> Components: Runtime / Checkpointing
> Affects Versions: 1.16.0
> Reporter: Yun Tang
> Priority: Major
>
>
> {code:java}
> 09:17:42,931 [flink-akka.actor.default-dispatcher-9] INFO
> org.apache.flink.runtime.jobmaster.JobMaster [] - Trying to
> recover from a global failure.
> java.lang.IllegalStateException: Cannot mark for checkpoint 17, already
> marked for checkpoint 16
> at
> org.apache.flink.runtime.operators.coordination.OperatorEventValve.markForCheckpoint(OperatorEventValve.java:113)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.checkpointCoordinatorInternal(OperatorCoordinatorHolder.java:302)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.lambda$checkpointCoordinator$0(OperatorCoordinatorHolder.java:230)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRunAsync$4(AkkaRpcActor.java:444)
> ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68)
> ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(AkkaRpcActor.java:444)
> ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:214)
> ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78)
> ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:164)
> ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at scala.PartialFunction.applyOrElse(PartialFunction.scala:123)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.actor.Actor.aroundReceive(Actor.scala:537)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.actor.Actor.aroundReceive$(Actor.scala:535)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.actor.ActorCell.invoke(ActorCell.scala:548)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.dispatch.Mailbox.run(Mailbox.scala:231)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.dispatch.Mailbox.exec(Mailbox.scala:243)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
> [?:1.8.0_292]
> at
> java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
> [?:1.8.0_292]
> at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
> [?:1.8.0_292]
> at
> java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
> [?:1.8.0_292]
> 09:17:42,932 [flink-akka.actor.default-dispatcher-9] INFO
> org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Job Flink
> Streaming Job (3bb746b6088ca587b3664599a7bae6e7) switched from state RUNNING
> to FAILING.
> org.apache.flink.runtime.JobException: Recovery is suppressed by
> FixedDelayRestartBackoffTimeStrategy(maxNumberRestartAttempts=5,
> backoffTimeMS=100)
> at
> org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.handleFailure(ExecutionFailureHandler.java:138)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.getGlobalFailureHandlingResult(ExecutionFailureHandler.java:101)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.scheduler.DefaultScheduler.handleGlobalFailure(DefaultScheduler.java:322)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.checkpointCoordinatorInternal(OperatorCoordinatorHolder.java:307)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.lambda$checkpointCoordinator$0(OperatorCoordinatorHolder.java:230)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRunAsync$4(AkkaRpcActor.java:444)
> ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68)
> ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(AkkaRpcActor.java:444)
> ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:214)
> ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78)
> ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:164)
> ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at scala.PartialFunction.applyOrElse(PartialFunction.scala:123)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.actor.Actor.aroundReceive(Actor.scala:537)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.actor.Actor.aroundReceive$(Actor.scala:535)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.actor.ActorCell.invoke(ActorCell.scala:548)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.dispatch.Mailbox.run(Mailbox.scala:231)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at akka.dispatch.Mailbox.exec(Mailbox.scala:243)
> [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT]
> at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
> [?:1.8.0_292]
> at
> java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
> [?:1.8.0_292]
> at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
> [?:1.8.0_292]
> at
> java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
> [?:1.8.0_292]
> Caused by: java.lang.IllegalStateException: Cannot mark for checkpoint 17,
> already marked for checkpoint 16
> at
> org.apache.flink.runtime.operators.coordination.OperatorEventValve.markForCheckpoint(OperatorEventValve.java:113)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.checkpointCoordinatorInternal(OperatorCoordinatorHolder.java:302)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> ... 27 more
> 09:17:42,930 [ Checkpoint Timer] WARN
> org.apache.flink.runtime.checkpoint.CheckpointFailureManager [] - Failed to
> trigger or complete checkpoint 17 for job 3bb746b6088ca587b3664599a7bae6e7.
> (0 consecutive failed attempts so far)
> org.apache.flink.runtime.checkpoint.CheckpointException: Trigger checkpoint
> failure.
> at
> org.apache.flink.runtime.checkpoint.CheckpointCoordinator.lambda$getCheckpointException$19(CheckpointCoordinator.java:2167)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> at java.util.Optional.orElseGet(Optional.java:267) ~[?:1.8.0_292]
> at
> org.apache.flink.runtime.checkpoint.CheckpointCoordinator.getCheckpointException(CheckpointCoordinator.java:2166)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.checkpoint.CheckpointCoordinator.onTriggerFailure(CheckpointCoordinator.java:938)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.checkpoint.CheckpointCoordinator.onTriggerFailure(CheckpointCoordinator.java:916)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.checkpoint.CheckpointCoordinator.lambda$startTriggeringCheckpoint$7(CheckpointCoordinator.java:642)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> at
> java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:836)
> [?:1.8.0_292]
> at
> java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:811)
> [?:1.8.0_292]
> at
> java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:456)
> [?:1.8.0_292]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> [?:1.8.0_292]
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> [?:1.8.0_292]
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
> [?:1.8.0_292]
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
> [?:1.8.0_292]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> [?:1.8.0_292]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> [?:1.8.0_292]
> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_292]
> Caused by: java.lang.IllegalStateException: Cannot mark for checkpoint 17,
> already marked for checkpoint 16
> at
> org.apache.flink.runtime.operators.coordination.OperatorEventValve.markForCheckpoint(OperatorEventValve.java:113)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.checkpointCoordinatorInternal(OperatorCoordinatorHolder.java:302)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.lambda$checkpointCoordinator$0(OperatorCoordinatorHolder.java:230)
> ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRunAsync$4(AkkaRpcActor.java:444)
> ~[?:?]
> at
> org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68)
> ~[?:?]
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(AkkaRpcActor.java:444)
> ~[?:?]
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:214)
> ~[?:?]
> at
> org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78)
> ~[?:?]
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:164)
> ~[?:?]
> at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24) ~[?:?]
> at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20) ~[?:?]
> at scala.PartialFunction.applyOrElse(PartialFunction.scala:123)
> ~[scala-library-2.12.7.jar:?]
> at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122)
> ~[scala-library-2.12.7.jar:?]
> at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20)
> ~[?:?]
> at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
> ~[scala-library-2.12.7.jar:?]
> at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
> ~[scala-library-2.12.7.jar:?]
> at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
> ~[scala-library-2.12.7.jar:?]
> at akka.actor.Actor.aroundReceive(Actor.scala:537) ~[?:?]
> at akka.actor.Actor.aroundReceive$(Actor.scala:535) ~[?:?]
> at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220)
> ~[?:?]
> at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580) ~[?:?]
> at akka.actor.ActorCell.invoke(ActorCell.scala:548) ~[?:?]
> at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270) ~[?:?]
> at akka.dispatch.Mailbox.run(Mailbox.scala:231) ~[?:?]
> at akka.dispatch.Mailbox.exec(Mailbox.scala:243) ~[?:?]
> at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
> ~[?:1.8.0_292]
> at
> java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
> ~[?:1.8.0_292]
> at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
> ~[?:1.8.0_292]
> at
> java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
> ~[?:1.8.0_292]
> {code}
> Instace:
> [https://dev.azure.com/apache-flink/apache-flink/_build/results?buildId=34771&view=logs&j=5c8e7682-d68f-54d1-16a2-a09310218a49&t=86f654fa-ab48-5c1a-25f4-7e7f6afb9bba]
>
> cc [~pnowojski] , [~becket_qin]
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
