Hi 这个调试可以在 IDEA 进行的。
另外你说的通过 web ui 提交没有问题。请问下，是同一个 savepoint 通过 flink run 提交有问题，通过 web ui 提交没有问题吗？如果是的，能否分享下你的操作过程和命令呢？ Best, Congxian Z-Z <[email protected]> 于2020年7月20日周一 上午11:33写道： > 这是taskmanager新报的一个错，还是跟之前一样，用cli提交报错，用webui提交就没问题： > 2020-07-20 03:29:25,959 WARN > org.apache.kafka.clients.consumer.ConsumerConfig > - The configuration 'value.serializer' was supplied > but isn't a known config. > 2020-07-20 03:29:25,959 INFO > org.apache.kafka.common.utils.AppInfoParser > - Kafka version : 0.11.0.2 > 2020-07-20 03:29:25,959 INFO > org.apache.kafka.common.utils.AppInfoParser > - Kafka commitId : 73be1e1168f91ee2 > 2020-07-20 03:29:25,974 ERROR > org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder > - Caught unexpected exception. > java.lang.ArrayIndexOutOfBoundsException: 0 > at > org.apache.flink.contrib.streaming.state.snapshot.RocksSnapshotUtil.hasMetaDataFollowsFlag(RocksSnapshotUtil.java:45) > at > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:223) > at > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168) > at > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151) > at > org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279) > at > org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548) > at > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288) > at > org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142) > at > org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121) > at > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304) > at > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131) > at > org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255) > at > org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006) > at > org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454) > at > org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94) > at > org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449) > at > org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461) > at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707) > at org.apache.flink.runtime.taskmanager.Task.run(Task.java:532) > at java.lang.Thread.run(Thread.java:748) > 2020-07-20 03:29:25,974 WARN > org.apache.flink.streaming.api.operators.BackendRestorerProcedure - > Exception while restoring keyed state backend for > StreamMap_caf773fe289bfdb867e0b4bd0c431c5f_(1/1) from alternative (1/1), > will retry while more alternatives are available. > org.apache.flink.runtime.state.BackendBuildingException: Caught unexpected > exception. > at > org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:336) > at > org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548) > at > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288) > at > org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142) > at > org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121) > at > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304) > at > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131) > at > org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255) > at > org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006) > at > org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454) > at > org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94) > at > org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449) > at > org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461) > at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707) > at org.apache.flink.runtime.taskmanager.Task.run(Task.java:532) > at java.lang.Thread.run(Thread.java:748) > Caused by: java.lang.ArrayIndexOutOfBoundsException: 0 > at > org.apache.flink.contrib.streaming.state.snapshot.RocksSnapshotUtil.hasMetaDataFollowsFlag(RocksSnapshotUtil.java:45) > at > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:223) > at > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168) > at > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151) > at > org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279) > ... 15 more > 2020-07-20 03:29:25,975 INFO > org.apache.kafka.clients.producer.KafkaProducer > - Closing the Kafka producer with timeoutMillis > = 9223372036854775807 ms. > 2020-07-20 03:29:25,979 INFO > org.apache.flink.runtime.taskmanager.Task > - Map -> Filter -> Sink: > Unnamed (1/1) (ed554502aa995fe53f1cf0cb8adf633c) switched from RUNNING to > FAILED. > java.lang.Exception: Exception while creating StreamOperatorStateContext. > at > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:191) > at > org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255) > at > org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006) > at > org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454) > at > org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94) > at > org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449) > at > org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461) > at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707) > at org.apache.flink.runtime.taskmanager.Task.run(Task.java:532) > at java.lang.Thread.run(Thread.java:748) > Caused by: org.apache.flink.util.FlinkException: Could not restore keyed > state backend for StreamMap_caf773fe289bfdb867e0b4bd0c431c5f_(1/1) from any > of the 1 provided restore options. > at > org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:135) > at > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304) > at > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131) > ... 9 more > Caused by: org.apache.flink.runtime.state.BackendBuildingException: Caught > unexpected exception. > at > org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:336) > at > org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548) > at > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288) > at > org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142) > at > org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121) > ... 11 more > Caused by: java.lang.ArrayIndexOutOfBoundsException: 0 > at > org.apache.flink.contrib.streaming.state.snapshot.RocksSnapshotUtil.hasMetaDataFollowsFlag(RocksSnapshotUtil.java:45) > at > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:223) > at > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168) > at > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151) > at > org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279) > > > > > > > > 谢谢回复： > 之前的savepoint都是通过RocksDBStateBackend生成的； > 这个savepoint我通过webui 提交任务就没问题，你是说在IDE上调试savepoint吗 > > > > > ------------------ 原始邮件 ------------------ > 发件人: > "user-zh" > < > [email protected]>; > 发送时间: 2020年7月19日(星期天) 晚上8:22 > 收件人: "user-zh"<[email protected]>; > > 主题: Re: Flink Cli 部署问题 > > > > Hi > > 从你给的这部分日志看，是恢复的时候遇到 EOF 了，这个比较奇怪 > 1 你之前的 savepoint 是使用 RocksDBStateBackend 生成的吗 > 2 你还有之前在 DFS 上的 savepoint 文件吗？可能需要结合 DFS 上的文件一起看一下这个问题怎么来的 > > Best, > Congxian > > > Z-Z <[email protected]> 于2020年7月17日周五 下午11:10写道： > > > Flink 1.10.0 ,taskmanager报错日志如下： > > > > > > 2020-07-17 15:06:43,913 ERROR > > > org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder&nbsp; > > - Caught unexpected exception. > > java.io.EOFException > > at > java.io.DataInputStream.readFully(DataInputStream.java:197) > > at > java.io.DataInputStream.readFully(DataInputStream.java:169) > > at > > > org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer.deserialize(BytePrimitiveArraySerializer.java:85) > > at > > > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:221) > > at > > > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168) > > at > > > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151) > > at > > > org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279) > > at > > > org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548) > > at > > > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288) > > at > > > org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142) > > at > > > org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121) > > at > > > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304) > > at > > > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131) > > at > > > org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255) > > at > > > org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006) > > at > > > org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454) > > at > > > org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94) > > at > > > org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449) > > at > > > org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461) > > at > org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707) > > at > org.apache.flink.runtime.taskmanager.Task.run(Task.java:532) > > at > java.lang.Thread.run(Thread.java:748) > > 2020-07-17 15:06:43,914 WARN&nbsp; > > > org.apache.flink.streaming.api.operators.BackendRestorerProcedure&nbsp; > - > > Exception while restoring keyed state backend for > > KeyedCoProcessOperator_00360b8021b192d84949201d4fea80f2_(1/1) from > > alternative (1/1), will retry while more alternatives are available. > > org.apache.flink.runtime.state.BackendBuildingException: Caught > unexpected > > exception. > > at > > > org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:336) > > at > > > org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548) > > at > > > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288) > > at > > > org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142) > > at > > > org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121) > > at > > > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304) > > at > > > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131) > > at > > > org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255) > > at > > > org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006) > > at > > > org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454) > > at > > > org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94) > > at > > > org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449) > > at > > > org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461) > > at > org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707) > > at > org.apache.flink.runtime.taskmanager.Task.run(Task.java:532) > > at > java.lang.Thread.run(Thread.java:748) > > Caused by: java.io.EOFException > > at > java.io.DataInputStream.readFully(DataInputStream.java:197) > > at > java.io.DataInputStream.readFully(DataInputStream.java:169) > > at > > > org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer.deserialize(BytePrimitiveArraySerializer.java:85) > > at > > > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:221) > > at > > > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168) > > at > > > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151) > > at > > > org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279) > > ... 15 more > > 2020-07-17 15:06:43,915 INFO&nbsp; > > org.apache.kafka.clients.producer.KafkaProducer&nbsp; &nbsp; > &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp;- Closing the Kafka > producer with timeoutMillis > > = 9223372036854775807 ms. > > 2020-07-17 15:06:43,918 INFO&nbsp; > > org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; > &nbsp; &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Co-Keyed-Process -&gt; Flat Map > > -&gt; Sink: Unnamed (1/1) (bb8f0a84e07ef90b1e11ca2825e0efab) > switched from > > RUNNING to FAILED. > > java.lang.Exception: Exception while creating > StreamOperatorStateContext. > > at > > > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:191) > > at > > > org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255) > > at > > > org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006) > > at > > > org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454) > > at > > > org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94) > > at > > > org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449) > > at > > > org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461) > > at > org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707) > > at > org.apache.flink.runtime.taskmanager.Task.run(Task.java:532) > > at > java.lang.Thread.run(Thread.java:748) > > Caused by: org.apache.flink.util.FlinkException: Could not restore > keyed > > state backend for > > KeyedCoProcessOperator_00360b8021b192d84949201d4fea80f2_(1/1) from > any of > > the 1 provided restore options. > > at > > > org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:135) > > at > > > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304) > > at > > > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131) > > ... 9 more > > Caused by: org.apache.flink.runtime.state.BackendBuildingException: > Caught > > unexpected exception. > > at > > > org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:336) > > at > > > org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548) > > at > > > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288) > > at > > > org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142) > > at > > > org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121) > > ... 11 more > > Caused by: java.io.EOFException > > at > java.io.DataInputStream.readFully(DataInputStream.java:197) > > at > java.io.DataInputStream.readFully(DataInputStream.java:169) > > at > > > org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer.deserialize(BytePrimitiveArraySerializer.java:85) > > at > > > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:221) > > at > > > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168) > > at > > > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151) > > at > > > org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279) > > ... 15 more > > 2020-07-17 15:06:43,919 INFO&nbsp; > > org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; > &nbsp; &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Freeing task resources for > > Co-Keyed-Process -&gt; Flat Map -&gt; Sink: Unnamed (1/1) > > (bb8f0a84e07ef90b1e11ca2825e0efab). > > 2020-07-17 15:06:43,919 INFO&nbsp; > > org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; > &nbsp; &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Ensuring all FileSystem streams > > are closed for task Co-Keyed-Process -&gt; Flat Map -&gt; > Sink: Unnamed > > (1/1) (bb8f0a84e07ef90b1e11ca2825e0efab) [FAILED] > > 2020-07-17 15:06:43,931 INFO&nbsp; > > org.apache.flink.runtime.taskexecutor.TaskExecutor&nbsp; > &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; - Un-registering task and sending > final execution > > state FAILED to JobManager for task Co-Keyed-Process -&gt; Flat > Map -&gt; > > Sink: Unnamed (1/1) bb8f0a84e07ef90b1e11ca2825e0efab. > > 2020-07-17 15:06:43,947 INFO&nbsp; > > org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; > &nbsp; &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Attempting to cancel task > > Source: Custom Source -&gt; Flat Map (1/1) > > (9cb8dcd4982223adcb6f007f1ffccdce). > > 2020-07-17 15:06:43,947 INFO&nbsp; > > org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; > &nbsp; &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Source: Custom Source -&gt; Flat > > Map (1/1) (9cb8dcd4982223adcb6f007f1ffccdce) switched from RUNNING to > > CANCELING. > > 2020-07-17 15:06:43,947 INFO&nbsp; > > org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; > &nbsp; &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Triggering cancellation of task > > code Source: Custom Source -&gt; Flat Map (1/1) > > (9cb8dcd4982223adcb6f007f1ffccdce). > > 2020-07-17 15:06:43,949 INFO&nbsp; > > org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; > &nbsp; &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Attempting to cancel task > > Source: Custom Source (1/1) (00621ff5d788d00c73ccaaea04717600). > > 2020-07-17 15:06:43,949 INFO&nbsp; > > org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; > &nbsp; &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Source: Custom Source (1/1) > > (00621ff5d788d00c73ccaaea04717600) switched from RUNNING to CANCELING. > > 2020-07-17 15:06:43,949 INFO&nbsp; > > org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; > &nbsp; &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Triggering cancellation of task > > code Source: Custom Source (1/1) (00621ff5d788d00c73ccaaea04717600). > > 2020-07-17 15:06:43,954 INFO&nbsp; > > org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; > &nbsp; &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Source: Custom Source -&gt; Flat > > Map (1/1) (9cb8dcd4982223adcb6f007f1ffccdce) switched from CANCELING > to > > CANCELED. > > 2020-07-17 15:06:43,954 INFO&nbsp; > > org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; > &nbsp; &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Freeing task resources for > > Source: Custom Source -&gt; Flat Map (1/1) > > (9cb8dcd4982223adcb6f007f1ffccdce). > > 2020-07-17 15:06:43,954 INFO&nbsp; > > org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; > &nbsp; &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Ensuring all FileSystem streams > > are closed for task Source: Custom Source -&gt; Flat Map (1/1) > > (9cb8dcd4982223adcb6f007f1ffccdce) [CANCELED] > > 2020-07-17 15:06:43,954 INFO&nbsp; > > org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; > &nbsp; &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Source: Custom Source (1/1) > > (00621ff5d788d00c73ccaaea04717600) switched from CANCELING to > CANCELED. > > 2020-07-17 15:06:43,955 INFO&nbsp; > > org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; > &nbsp; &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Freeing task resources for > > Source: Custom Source (1/1) (00621ff5d788d00c73ccaaea04717600). > > 2020-07-17 15:06:43,954 INFO&nbsp; > > org.apache.flink.runtime.taskexecutor.TaskExecutor&nbsp; > &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; - Un-registering task and sending > final execution > > state CANCELED to JobManager for task Source: Custom Source -&gt; > Flat Map > > (1/1) 9cb8dcd4982223adcb6f007f1ffccdce. > > 2020-07-17 15:06:43,962 INFO&nbsp; > > org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; > &nbsp; &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Ensuring all FileSystem streams > > are closed for task Source: Custom Source (1/1) > > (00621ff5d788d00c73ccaaea04717600) [CANCELED] > > 2020-07-17 15:06:43,962 INFO&nbsp; > > org.apache.flink.runtime.taskexecutor.TaskExecutor&nbsp; > &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; - Un-registering task and sending > final execution > > state CANCELED to JobManager for task Source: Custom Source (1/1) > > 00621ff5d788d00c73ccaaea04717600. > > 2020-07-17 15:06:44,077 WARN&nbsp; > > org.apache.kafka.clients.consumer.ConsumerConfig&nbsp; &nbsp; > &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; - The configuration ' > transaction.timeout.ms' was > > supplied but isn't a known config. > > 2020-07-17 15:06:44,077 WARN&nbsp; > > org.apache.kafka.clients.consumer.ConsumerConfig&nbsp; &nbsp; > &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; - The configuration 'key.serializer' > was supplied but > > isn't a known config. > > 2020-07-17 15:06:44,077 WARN&nbsp; > > org.apache.kafka.clients.consumer.ConsumerConfig&nbsp; &nbsp; > &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; - The configuration > 'value.serializer' was supplied > > but isn't a known config. > > 2020-07-17 15:06:44,077 INFO&nbsp; > > org.apache.kafka.common.utils.AppInfoParser&nbsp; &nbsp; > &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Kafka version : 0.11.0.2 > > 2020-07-17 15:06:44,077 INFO&nbsp; > > org.apache.kafka.common.utils.AppInfoParser&nbsp; &nbsp; > &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Kafka commitId : 73be1e1168f91ee2 > > 2020-07-17 15:06:44,077 WARN&nbsp; > > org.apache.kafka.common.utils.AppInfoParser&nbsp; &nbsp; > &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Error registering AppInfo mbean > > javax.management.InstanceAlreadyExistsException: > > kafka.consumer:type=app-info,id=consumer-3 > > at > com.sun.jmx.mbeanserver.Repository.addMBean(Repository.java:437) > > at > > > com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerWithRepository(DefaultMBeanServerInterceptor.java:1898) > > at > > > com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerDynamicMBean(DefaultMBeanServerInterceptor.java:966) > > at > > > com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerObject(DefaultMBeanServerInterceptor.java:900) > > at > > > com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerMBean(DefaultMBeanServerInterceptor.java:324) > > at > > > com.sun.jmx.mbeanserver.JmxMBeanServer.registerMBean(JmxMBeanServer.java:522) > > at > > > org.apache.kafka.common.utils.AppInfoParser.registerAppInfo(AppInfoParser.java:58) > > at > > > org.apache.kafka.clients.consumer.KafkaConsumer.<init&gt;(KafkaConsumer.java:757) > > at > > > org.apache.kafka.clients.consumer.KafkaConsumer.<init&gt;(KafkaConsumer.java:633) > > at > > > org.apache.kafka.clients.consumer.KafkaConsumer.<init&gt;(KafkaConsumer.java:615) > > at > > > org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.getConsumer(KafkaConsumerThread.java:502) > > at > > > org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.run(KafkaConsumerThread.java:181) > > 2020-07-17 15:06:44,079 INFO&nbsp; > > org.apache.kafka.common.utils.AppInfoParser&nbsp; &nbsp; > &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Kafka version : 0.11.0.2 > > 2020-07-17 15:06:44,079 INFO&nbsp; > > org.apache.kafka.common.utils.AppInfoParser&nbsp; &nbsp; > &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Kafka commitId : 73be1e1168f91ee2 > > 2020-07-17 15:06:44,079 WARN&nbsp; > > org.apache.kafka.common.utils.AppInfoParser&nbsp; &nbsp; > &nbsp; &nbsp; > > &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- > Error registering AppInfo mbean > > javax.management.InstanceAlreadyExistsException: > > kafka.consumer:type=app-info,id=consumer-4 > > at > com.sun.jmx.mbeanserver.Repository.addMBean(Repository.java:437) > > at > > > com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerWithRepository(DefaultMBeanServerInterceptor.java:1898) > > at > > > com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerDynamicMBean(DefaultMBeanServerInterceptor.java:966) > > at > > > com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerObject(DefaultMBeanServerInterceptor.java:900) > > at > > > com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerMBean(DefaultMBeanServerInterceptor.java:324) > > at > > > com.sun.jmx.mbeanserver.JmxMBeanServer.registerMBean(JmxMBeanServer.java:522) > > at > > > org.apache.kafka.common.utils.AppInfoParser.registerAppInfo(AppInfoParser.java:58) > > at > > > org.apache.kafka.clients.consumer.KafkaConsumer.<init&gt;(KafkaConsumer.java:757) > > at > > > org.apache.kafka.clients.consumer.KafkaConsumer.<init&gt;(KafkaConsumer.java:633) > > at > > > org.apache.kafka.clients.consumer.KafkaConsumer.<init&gt;(KafkaConsumer.java:615) > > at > > > org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.getConsumer(KafkaConsumerThread.java:502) > > at > > > org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.run(KafkaConsumerThread.java:181) > > > > > > > > > > > > ------------------&nbsp;原始邮件&nbsp;------------------ > > 发件人: > > > "user-zh" > > > < > > [email protected]&gt;; > > 发送时间:&nbsp;2020年7月17日(星期五) 晚上10:52 > > 收件人:&nbsp;"user-zh"<[email protected]&gt;; > > > > 主题:&nbsp;Re: Flink Cli 部署问题 > > > > > > > > Hi > > > > 请问你使用哪个版本的 Flink 呢？能否分享一下&nbsp; Co-Process (1/1) > > (d0309f26a545e74643382ed3f758269b) 这个 tm 的 log 呢？从上面给的日志看，应该是在 > 083f69d029de > > 这台机器上。 > > > > Best, > > Congxian > > > > > > Z-Z <[email protected]&gt; 于2020年7月17日周五 下午6:22写道： > > > > &gt; > > > 大家好，我在部署的时候发现了一个问题，我通过restAPI接口停掉了一个任务并保存了它的savepoint(步骤：/jobs/overview > > &gt; ---&amp;gt; /jobs/{jobid}/savepoints ---&amp;gt; > > &gt; > > > /jobs/{jobid}/savepoints/{triggerid})，但我通过flink命令带上savepoint部署任务时会报错，但通过webui上传jar并带上savepoint就不会报错，报错堆栈如下： > > &gt; 2020-07-17 09:51:48,925 INFO&amp;nbsp; > > &gt; > > > org.apache.flink.runtime.resourcemanager.StandaloneResourceManager&amp;nbsp; > > - > > &gt; Request slot with profile ResourceProfile{UNKNOWN} for job > > &gt; 7639673873b707aa86c4387aa7b4aac3 with allocation id > > &gt; e8865cdbfe4c3c33099c7112bc2e3231. > > &gt; 2020-07-17 09:51:48,952 INFO&amp;nbsp; > > &gt; > org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp; > > &amp;nbsp; &amp;nbsp; > > &gt; &amp;nbsp; - Source: Custom Source -&amp;gt; Filter > (1/1) > > &gt; (1177659bff014e8dbc3f0508055d4307) switched from SCHEDULED to > > DEPLOYING. > > &gt; 2020-07-17 09:51:48,952 INFO&amp;nbsp; > > &gt; > org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp; > > &amp;nbsp; &amp;nbsp; > > &gt; &amp;nbsp; - Deploying Source: Custom Source > -&amp;gt; Filter (1/1) > > (attempt #0) to > > &gt; e63d829deafc144cd82efd73979dd056 @ 083f69d029de > (dataPort=35758) > > &gt; 2020-07-17 09:51:48,953 INFO&amp;nbsp; > > &gt; > org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp; > > &amp;nbsp; &amp;nbsp; > > &gt; &amp;nbsp; - Source: Custom Source (1/1) > > (141f0dc22b624b39e21127f637ba63c2) > > &gt; switched from SCHEDULED to DEPLOYING. > > &gt; 2020-07-17 09:51:48,953 INFO&amp;nbsp; > > &gt; > org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp; > > &amp;nbsp; &amp;nbsp; > > &gt; &amp;nbsp; - Deploying Source: Custom Source (1/1) > (attempt #0) to > > &gt; e63d829deafc144cd82efd73979dd056 @ 083f69d029de > (dataPort=35758) > > &gt; 2020-07-17 09:51:48,954 INFO&amp;nbsp; > > &gt; > org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp; > > &amp;nbsp; &amp;nbsp; > > &gt; &amp;nbsp; - Source: Custom Source (1/1) > > (274b3df03e1fab627059c1a78e4a26da) > > &gt; switched from SCHEDULED to DEPLOYING. > > &gt; 2020-07-17 09:51:48,954 INFO&amp;nbsp; > > &gt; > org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp; > > &amp;nbsp; &amp;nbsp; > > &gt; &amp;nbsp; - Deploying Source: Custom Source (1/1) > (attempt #0) to > > &gt; e63d829deafc144cd82efd73979dd056 @ 083f69d029de > (dataPort=35758) > > &gt; 2020-07-17 09:51:48,954 INFO&amp;nbsp; > > &gt; > org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp; > > &amp;nbsp; &amp;nbsp; > > &gt; &amp;nbsp; - Co-Process (1/1) > (d0309f26a545e74643382ed3f758269b) > > switched from > > &gt; SCHEDULED to DEPLOYING. > > &gt; 2020-07-17 09:51:48,954 INFO&amp;nbsp; > > &gt; > org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp; > > &amp;nbsp; &amp;nbsp; > > &gt; &amp;nbsp; - Deploying Co-Process (1/1) (attempt #0) to > > &gt; e63d829deafc144cd82efd73979dd056 @ 083f69d029de > (dataPort=35758) > > &gt; 2020-07-17 09:51:48,955 INFO&amp;nbsp; > > &gt; > org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp; > > &amp;nbsp; &amp;nbsp; > > &gt; &amp;nbsp; - Co-Process -&amp;gt; (Sink: Unnamed, > Sink: Unnamed) (1/1) > > &gt; (618b75fcf5ea05fb5c6487bec6426e31) switched from SCHEDULED to > > DEPLOYING. > > &gt; 2020-07-17 09:51:48,955 INFO&amp;nbsp; > > &gt; > org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp; > > &amp;nbsp; &amp;nbsp; > > &gt; &amp;nbsp; - Deploying Co-Process -&amp;gt; (Sink: > Unnamed, Sink: > > Unnamed) (1/1) > > &gt; (attempt #0) to e63d829deafc144cd82efd73979dd056 @ > 083f69d029de > > &gt; (dataPort=35758) > > &gt; 2020-07-17 09:51:49,346 INFO&amp;nbsp; > > &gt; > org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp; > > &amp;nbsp; &amp;nbsp; > > &gt; &amp;nbsp; - Co-Process -&amp;gt; (Sink: Unnamed, > Sink: Unnamed) (1/1) > > &gt; (618b75fcf5ea05fb5c6487bec6426e31) switched from DEPLOYING > to RUNNING. > > &gt; 2020-07-17 09:51:49,370 INFO&amp;nbsp; > > &gt; > org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp; > > &amp;nbsp; &amp;nbsp; > > &gt; &amp;nbsp; - Source: Custom Source (1/1) > > (274b3df03e1fab627059c1a78e4a26da) > > &gt; switched from DEPLOYING to RUNNING. > > &gt; 2020-07-17 09:51:49,370 INFO&amp;nbsp; > > &gt; > org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp; > > &amp;nbsp; &amp;nbsp; > > &gt; &amp;nbsp; - Source: Custom Source (1/1) > > (141f0dc22b624b39e21127f637ba63c2) > > &gt; switched from DEPLOYING to RUNNING. > > &gt; 2020-07-17 09:51:49,377 INFO&amp;nbsp; > > &gt; > org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp; > > &amp;nbsp; &amp;nbsp; > > &gt; &amp;nbsp; - Co-Process (1/1) > (d0309f26a545e74643382ed3f758269b) > > switched from > > &gt; DEPLOYING to RUNNING. > > &gt; 2020-07-17 09:51:49,377 INFO&amp;nbsp; > > &gt; > org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp; > > &amp;nbsp; &amp;nbsp; > > &gt; &amp;nbsp; - Source: Custom Source -&amp;gt; Filter > (1/1) > > &gt; (1177659bff014e8dbc3f0508055d4307) switched from DEPLOYING > to RUNNING. > > &gt; 2020-07-17 09:51:49,493 INFO&amp;nbsp; > > &gt; > org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp; > > &amp;nbsp; &amp;nbsp; > > &gt; &amp;nbsp; - Co-Process (1/1) > (d0309f26a545e74643382ed3f758269b) > > switched from > > &gt; RUNNING to FAILED. > > &gt; java.lang.Exception: Exception while creating > > StreamOperatorStateContext. > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:191) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > org.apache.flink.runtime.taskmanager.Task.run(Task.java:532) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > java.lang.Thread.run(Thread.java:748) > > &gt; Caused by: org.apache.flink.util.FlinkException: Could not > restore > > keyed > > &gt; state backend for > > &gt; > LegacyKeyedCoProcessOperator_65e7116c7aa972ad18a796ae22bd6327_(1/1) > > from > > &gt; any of the 1 provided restore options. > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:135) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > ... 9 more > > &gt; Caused by: > org.apache.flink.runtime.state.BackendBuildingException: > > Caught > > &gt; unexpected exception. > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:336) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > ... 11 more > > &gt; Caused by: java.io.EOFException > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > java.io.DataInputStream.readFully(DataInputStream.java:197) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > java.io.DataInputStream.readFully(DataInputStream.java:169) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer.deserialize(BytePrimitiveArraySerializer.java:85) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:221) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > at > > &gt; > > > org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279) > > > &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; > ... 15 more
