eric created HUDI-6617:
--------------------------
Summary: Fix HoodieRecordDelegate NotSerializableException
Key: HUDI-6617
URL: https://issues.apache.org/jira/browse/HUDI-6617
Project: Apache Hudi
Issue Type: Bug
Components: code-quality
Reporter: eric
2023-07-31 12:26:31,586 INFO
org.apache.flink.runtime.checkpoint.CheckpointCoordinator [] - Triggering
checkpoint 1 (type=CheckpointType{name='Checkpoint',
sharingFilesStrategy=FORWARD_BACKWARD}) @ 1690777591567 for job
48b9d3ed6ce36311ecf93c6fa30f65df.
2023-07-31 12:26:31,621 INFO
org.apache.hudi.sink.StreamWriteOperatorCoordinator [] - Executor
executes action [taking checkpoint 1] success!
2023-07-31 12:26:35,279 INFO
org.apache.flink.runtime.executiongraph.ExecutionGraph [] - bucket_write:
default_database.random_hudi (10/10)
(5adfdfb477c99ae4f6ff42a9edce4a64_14abfc6a642f6076d66d2dceb9ca3dd9_9_0)
switched from RUNNING to FAILED on container_e08_1689217840902_0081_01_000003 @
host-10-19-29-165 (dataPort=36699).
java.io.IOException: Could not perform checkpoint 1 for operator bucket_write:
default_database.random_hudi (10/10)#0.
at
org.apache.flink.streaming.runtime.tasks.StreamTask.triggerCheckpointOnBarrier(StreamTask.java:1238)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.io.checkpointing.CheckpointBarrierHandler.notifyCheckpoint(CheckpointBarrierHandler.java:147)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.io.checkpointing.SingleCheckpointBarrierHandler.triggerCheckpoint(SingleCheckpointBarrierHandler.java:287)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.io.checkpointing.SingleCheckpointBarrierHandler.access$100(SingleCheckpointBarrierHandler.java:64)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.io.checkpointing.SingleCheckpointBarrierHandler$ControllerImpl.triggerGlobalCheckpoint(SingleCheckpointBarrierHandler.java:488)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.io.checkpointing.AbstractAlignedBarrierHandlerState.triggerGlobalCheckpoint(AbstractAlignedBarrierHandlerState.java:74)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.io.checkpointing.AbstractAlignedBarrierHandlerState.barrierReceived(AbstractAlignedBarrierHandlerState.java:66)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.io.checkpointing.SingleCheckpointBarrierHandler.lambda$processBarrier$2(SingleCheckpointBarrierHandler.java:234)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.io.checkpointing.SingleCheckpointBarrierHandler.markCheckpointAlignedAndTransformState(SingleCheckpointBarrierHandler.java:262)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.io.checkpointing.SingleCheckpointBarrierHandler.processBarrier(SingleCheckpointBarrierHandler.java:231)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.io.checkpointing.CheckpointedInputGate.handleEvent(CheckpointedInputGate.java:181)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.io.checkpointing.CheckpointedInputGate.pollNext(CheckpointedInputGate.java:159)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.io.AbstractStreamTaskNetworkInput.emitNext(AbstractStreamTaskNetworkInput.java:110)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.io.StreamOneInputProcessor.processInput(StreamOneInputProcessor.java:65)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.processInput(StreamTask.java:542)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:231)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.runMailboxLoop(StreamTask.java:831)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:780)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.runtime.taskmanager.Task.runWithSystemExitMonitoring(Task.java:935)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.runtime.taskmanager.Task.restoreAndInvoke(Task.java:914)
~[flink-dist-1.16.1.jar:1.16.1]
at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:728)
~[flink-dist-1.16.1.jar:1.16.1]
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:550)
~[flink-dist-1.16.1.jar:1.16.1]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_271]
Caused by: org.apache.flink.runtime.checkpoint.CheckpointException: Could not
complete snapshot 1 for operator bucket_write: default_database.random_hudi
(10/10)#0. Failure reason: Checkpoint was declined.
at
org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.snapshotState(StreamOperatorStateHandler.java:269)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.snapshotState(StreamOperatorStateHandler.java:173)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.api.operators.AbstractStreamOperator.snapshotState(AbstractStreamOperator.java:345)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.RegularOperatorChain.checkpointStreamOperator(RegularOperatorChain.java:228)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.RegularOperatorChain.buildOperatorSnapshotFutures(RegularOperatorChain.java:213)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.RegularOperatorChain.snapshotState(RegularOperatorChain.java:192)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.takeSnapshotSync(SubtaskCheckpointCoordinatorImpl.java:726)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.checkpointState(SubtaskCheckpointCoordinatorImpl.java:363)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$performCheckpoint$13(StreamTask.java:1281)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.runThrowing(StreamTaskActionExecutor.java:50)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.performCheckpoint(StreamTask.java:1269)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.triggerCheckpointOnBarrier(StreamTask.java:1226)
~[flink-dist-1.16.1.jar:1.16.1]
... 22 more
Caused by: org.apache.flink.util.FlinkRuntimeException: Cannot serialize
operator event
at
org.apache.flink.streaming.runtime.tasks.OperatorEventDispatcherImpl$OperatorEventGatewayImpl.sendEventToCoordinator(OperatorEventDispatcherImpl.java:117)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.hudi.sink.StreamWriteFunction.flushRemaining(StreamWriteFunction.java:484)
~[hudi-flink1.16-bundle-0.14.0-SNAPSHOT.jar:0.14.0-SNAPSHOT]
at
org.apache.hudi.sink.StreamWriteFunction.snapshotState(StreamWriteFunction.java:137)
~[hudi-flink1.16-bundle-0.14.0-SNAPSHOT.jar:0.14.0-SNAPSHOT]
at
org.apache.hudi.sink.bucket.BucketStreamWriteFunction.snapshotState(BucketStreamWriteFunction.java:101)
~[hudi-flink1.16-bundle-0.14.0-SNAPSHOT.jar:0.14.0-SNAPSHOT]
at
org.apache.hudi.sink.common.AbstractStreamWriteFunction.snapshotState(AbstractStreamWriteFunction.java:167)
~[hudi-flink1.16-bundle-0.14.0-SNAPSHOT.jar:0.14.0-SNAPSHOT]
at
org.apache.flink.streaming.util.functions.StreamingFunctionUtils.trySnapshotFunctionState(StreamingFunctionUtils.java:118)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.util.functions.StreamingFunctionUtils.snapshotFunctionState(StreamingFunctionUtils.java:99)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator.snapshotState(AbstractUdfStreamOperator.java:87)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.snapshotState(StreamOperatorStateHandler.java:222)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.snapshotState(StreamOperatorStateHandler.java:173)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.api.operators.AbstractStreamOperator.snapshotState(AbstractStreamOperator.java:345)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.RegularOperatorChain.checkpointStreamOperator(RegularOperatorChain.java:228)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.RegularOperatorChain.buildOperatorSnapshotFutures(RegularOperatorChain.java:213)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.RegularOperatorChain.snapshotState(RegularOperatorChain.java:192)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.takeSnapshotSync(SubtaskCheckpointCoordinatorImpl.java:726)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.checkpointState(SubtaskCheckpointCoordinatorImpl.java:363)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$performCheckpoint$13(StreamTask.java:1281)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.runThrowing(StreamTaskActionExecutor.java:50)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.performCheckpoint(StreamTask.java:1269)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.triggerCheckpointOnBarrier(StreamTask.java:1226)
~[flink-dist-1.16.1.jar:1.16.1]
... 22 more
Caused by: java.io.NotSerializableException:
org.apache.hudi.common.model.HoodieRecordDelegate
at
java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1184)
~[?:1.8.0_271]
at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348)
~[?:1.8.0_271]
at java.util.ArrayList.writeObject(ArrayList.java:768) ~[?:1.8.0_271]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
~[?:1.8.0_271]
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
~[?:1.8.0_271]
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:1.8.0_271]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_271]
at
java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:1155)
~[?:1.8.0_271]
at
java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1496)
~[?:1.8.0_271]
at
java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
~[?:1.8.0_271]
at
java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
~[?:1.8.0_271]
at
java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
~[?:1.8.0_271]
at
java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
~[?:1.8.0_271]
at
java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
~[?:1.8.0_271]
at
java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
~[?:1.8.0_271]
at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348)
~[?:1.8.0_271]
at java.util.ArrayList.writeObject(ArrayList.java:768) ~[?:1.8.0_271]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
~[?:1.8.0_271]
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
~[?:1.8.0_271]
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:1.8.0_271]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_271]
at
java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:1155)
~[?:1.8.0_271]
at
java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1496)
~[?:1.8.0_271]
at
java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
~[?:1.8.0_271]
at
java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
~[?:1.8.0_271]
at
java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
~[?:1.8.0_271]
at
java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
~[?:1.8.0_271]
at
java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
~[?:1.8.0_271]
at
java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
~[?:1.8.0_271]
at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348)
~[?:1.8.0_271]
at
org.apache.flink.util.InstantiationUtil.serializeObject(InstantiationUtil.java:632)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.util.SerializedValue.<init>(SerializedValue.java:62)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.OperatorEventDispatcherImpl$OperatorEventGatewayImpl.sendEventToCoordinator(OperatorEventDispatcherImpl.java:113)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.hudi.sink.StreamWriteFunction.flushRemaining(StreamWriteFunction.java:484)
~[hudi-flink1.16-bundle-0.14.0-SNAPSHOT.jar:0.14.0-SNAPSHOT]
at
org.apache.hudi.sink.StreamWriteFunction.snapshotState(StreamWriteFunction.java:137)
~[hudi-flink1.16-bundle-0.14.0-SNAPSHOT.jar:0.14.0-SNAPSHOT]
at
org.apache.hudi.sink.bucket.BucketStreamWriteFunction.snapshotState(BucketStreamWriteFunction.java:101)
~[hudi-flink1.16-bundle-0.14.0-SNAPSHOT.jar:0.14.0-SNAPSHOT]
at
org.apache.hudi.sink.common.AbstractStreamWriteFunction.snapshotState(AbstractStreamWriteFunction.java:167)
~[hudi-flink1.16-bundle-0.14.0-SNAPSHOT.jar:0.14.0-SNAPSHOT]
at
org.apache.flink.streaming.util.functions.StreamingFunctionUtils.trySnapshotFunctionState(StreamingFunctionUtils.java:118)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.util.functions.StreamingFunctionUtils.snapshotFunctionState(StreamingFunctionUtils.java:99)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator.snapshotState(AbstractUdfStreamOperator.java:87)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.snapshotState(StreamOperatorStateHandler.java:222)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.snapshotState(StreamOperatorStateHandler.java:173)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.api.operators.AbstractStreamOperator.snapshotState(AbstractStreamOperator.java:345)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.RegularOperatorChain.checkpointStreamOperator(RegularOperatorChain.java:228)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.RegularOperatorChain.buildOperatorSnapshotFutures(RegularOperatorChain.java:213)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.RegularOperatorChain.snapshotState(RegularOperatorChain.java:192)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.takeSnapshotSync(SubtaskCheckpointCoordinatorImpl.java:726)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.SubtaskCheckpointCoordinatorImpl.checkpointState(SubtaskCheckpointCoordinatorImpl.java:363)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$performCheckpoint$13(StreamTask.java:1281)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.runThrowing(StreamTaskActionExecutor.java:50)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.performCheckpoint(StreamTask.java:1269)
~[flink-dist-1.16.1.jar:1.16.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.triggerCheckpointOnBarrier(StreamTask.java:1226)
~[flink-dist-1.16.1.jar:1.16.1]
... 22 more
2023-07-31 12:26:35,357 INFO org.apache.flink.runtime.jobmaster.JobMaster
[] - 23 tasks will be restarted to recover the failed task
5adfdfb477c99ae4f6ff42a9edce4a64_14abfc6a642f6076d66d2dceb9ca3dd9_9_0.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
