[ https://issues.apache.org/jira/browse/HUDI-3638?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Ethan Guo closed HUDI-3638. --------------------------- Resolution: Fixed > Task not serializable due to ZookeeperBasedLockProvider in clean action > ----------------------------------------------------------------------- > > Key: HUDI-3638 > URL: https://issues.apache.org/jira/browse/HUDI-3638 > Project: Apache Hudi > Issue Type: Task > Reporter: Ethan Guo > Assignee: Ethan Guo > Priority: Blocker > Labels: pull-request-available > Fix For: 0.11.0 > > > > {code:java} > 22/03/14 10:58:05 WARN CleanActionExecutor: Failed to perform previous clean > operation, instant: [==>20220314105144254__clean__REQUESTED] > org.apache.spark.SparkException: Task not serializable > at > org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:416) > at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:406) > at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:162) > at org.apache.spark.SparkContext.clean(SparkContext.scala:2477) > at org.apache.spark.rdd.RDD.$anonfun$mapPartitions$1(RDD.scala:860) > at > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) > at > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) > at org.apache.spark.rdd.RDD.withScope(RDD.scala:414) > at org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:859) > at > org.apache.spark.api.java.JavaRDDLike.mapPartitionsToPair(JavaRDDLike.scala:188) > at > org.apache.spark.api.java.JavaRDDLike.mapPartitionsToPair$(JavaRDDLike.scala:183) > at > org.apache.spark.api.java.AbstractJavaRDDLike.mapPartitionsToPair(JavaRDDLike.scala:45) > at > org.apache.hudi.client.common.HoodieSparkEngineContext.mapPartitionsToPairAndReduceByKey(HoodieSparkEngineContext.java:116) > at > org.apache.hudi.table.action.clean.CleanActionExecutor.clean(CleanActionExecutor.java:141) > at > org.apache.hudi.table.action.clean.CleanActionExecutor.runClean(CleanActionExecutor.java:197) > at > org.apache.hudi.table.action.clean.CleanActionExecutor.runPendingClean(CleanActionExecutor.java:176) > at > org.apache.hudi.table.action.clean.CleanActionExecutor.lambda$execute$6(CleanActionExecutor.java:238) > at java.util.ArrayList.forEach(ArrayList.java:1259) > at > org.apache.hudi.table.action.clean.CleanActionExecutor.execute(CleanActionExecutor.java:232) > at > org.apache.hudi.table.HoodieSparkCopyOnWriteTable.clean(HoodieSparkCopyOnWriteTable.java:339) > at > org.apache.hudi.client.BaseHoodieWriteClient.clean(BaseHoodieWriteClient.java:780) > at > org.apache.hudi.client.BaseHoodieWriteClient.clean(BaseHoodieWriteClient.java:737) > at > org.apache.hudi.async.AsyncCleanerService.lambda$startService$0(AsyncCleanerService.java:55) > at > java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at java.lang.Thread.run(Thread.java:748) > Caused by: java.io.NotSerializableException: > org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider > Serialization stack: > - object not serializable (class: > org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider, value: > org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider@2e4da831) > - field (class: org.apache.hudi.client.transaction.lock.LockManager, > name: lockProvider, type: interface org.apache.hudi.common.lock.LockProvider) > - object (class org.apache.hudi.client.transaction.lock.LockManager, > org.apache.hudi.client.transaction.lock.LockManager@60d74cfd) > - field (class: org.apache.hudi.client.transaction.TransactionManager, > name: lockManager, type: class > org.apache.hudi.client.transaction.lock.LockManager) > - object (class org.apache.hudi.client.transaction.TransactionManager, > org.apache.hudi.client.transaction.TransactionManager@54e830d4) > - field (class: org.apache.hudi.table.action.clean.CleanActionExecutor, > name: txnManager, type: class > org.apache.hudi.client.transaction.TransactionManager) > - object (class org.apache.hudi.table.action.clean.CleanActionExecutor, > org.apache.hudi.table.action.clean.CleanActionExecutor@71c368a0) > - element of array (index: 0) > - array (class [Ljava.lang.Object;, size 1) > - field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, > type: class [Ljava.lang.Object;) > - object (class java.lang.invoke.SerializedLambda, > SerializedLambda[capturingClass=class > org.apache.hudi.table.action.clean.CleanActionExecutor, > functionalInterfaceMethod=org/apache/hudi/common/function/SerializablePairFlatMapFunction.call:(Ljava/lang/Object;)Ljava/util/stream/Stream;, > implementation=invokeSpecial > org/apache/hudi/table/action/clean/CleanActionExecutor.lambda$clean$59ddfded$1:(Ljava/util/Iterator;)Ljava/util/stream/Stream;, > instantiatedMethodType=(Ljava/util/Iterator;)Ljava/util/stream/Stream;, > numCaptured=1]) > - writeReplace data (class: java.lang.invoke.SerializedLambda) > - object (class > org.apache.hudi.table.action.clean.CleanActionExecutor$$Lambda$3706/863841076, > > org.apache.hudi.table.action.clean.CleanActionExecutor$$Lambda$3706/863841076@5e1f7ca3) > - element of array (index: 0) > - array (class [Ljava.lang.Object;, size 1) > - field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, > type: class [Ljava.lang.Object;) > - object (class java.lang.invoke.SerializedLambda, > SerializedLambda[capturingClass=class > org.apache.hudi.client.common.HoodieSparkEngineContext, > functionalInterfaceMethod=org/apache/spark/api/java/function/PairFlatMapFunction.call:(Ljava/lang/Object;)Ljava/util/Iterator;, > implementation=invokeStatic > org/apache/hudi/client/common/HoodieSparkEngineContext.lambda$mapPartitionsToPairAndReduceByKey$4e9dbab9$1:(Lorg/apache/hudi/common/function/SerializablePairFlatMapFunction;Ljava/util/Iterator;)Ljava/util/Iterator;, > instantiatedMethodType=(Ljava/util/Iterator;)Ljava/util/Iterator;, > numCaptured=1]) > - writeReplace data (class: java.lang.invoke.SerializedLambda) > - object (class > org.apache.hudi.client.common.HoodieSparkEngineContext$$Lambda$3709/1146198397, > > org.apache.hudi.client.common.HoodieSparkEngineContext$$Lambda$3709/1146198397@7aff92c3) > - element of array (index: 0) > - array (class [Ljava.lang.Object;, size 1) > - field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, > type: class [Ljava.lang.Object;) > - object (class java.lang.invoke.SerializedLambda, > SerializedLambda[capturingClass=interface > org.apache.spark.api.java.JavaRDDLike, > functionalInterfaceMethod=scala/Function1.apply:(Ljava/lang/Object;)Ljava/lang/Object;, > implementation=invokeStatic > org/apache/spark/api/java/JavaRDDLike.$anonfun$mapPartitionsToPair$1:(Lorg/apache/spark/api/java/function/PairFlatMapFunction;Lscala/collection/Iterator;)Lscala/collection/Iterator;, > > instantiatedMethodType=(Lscala/collection/Iterator;)Lscala/collection/Iterator;, > numCaptured=1]) > - writeReplace data (class: java.lang.invoke.SerializedLambda) > - object (class > org.apache.spark.api.java.JavaRDDLike$$Lambda$3710/1144734313, > org.apache.spark.api.java.JavaRDDLike$$Lambda$3710/1144734313@4a33e3f8) > at > org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:41) > at > org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:47) > at > org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:101) > at > org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:413) > ... 26 more {code} -- This message was sent by Atlassian Jira (v8.20.1#820001)