[
https://issues.apache.org/jira/browse/HUDI-3587?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
sivabalan narayanan closed HUDI-3587.
-------------------------------------
Assignee: sivabalan narayanan
Resolution: Fixed
> SparkUpgradeDowngradeHelper not serialization exception
> -------------------------------------------------------
>
> Key: HUDI-3587
> URL: https://issues.apache.org/jira/browse/HUDI-3587
> Project: Apache Hudi
> Issue Type: Bug
> Reporter: sivabalan narayanan
> Assignee: sivabalan narayanan
> Priority: Critical
> Labels: pull-request-available
> Fix For: 0.11.0
>
>
> {code:java}
> [AWS CodeBuild Plugin] - object (class
> org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter$$Lambda$138/1591629766,
> org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter$$Lambda$138/1591629766@18e13131)
> [AWS CodeBuild Plugin] - field (class:
> org.apache.spark.api.java.JavaPairRDD$$anonfun$toScalaFunction$1, name:
> fun$1, type: interface org.apache.spark.api.java.function.Function)
> [AWS CodeBuild Plugin] - object (class
> org.apache.spark.api.java.JavaPairRDD$$anonfun$toScalaFunction$1, <function1>)
> [AWS CodeBuild Plugin] at
> org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:40)
> [AWS CodeBuild Plugin] at
> org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:46)
> [AWS CodeBuild Plugin] at
> org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:100)
> [AWS CodeBuild Plugin] at
> org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:400)
> [AWS CodeBuild Plugin] ... 20 more
> [AWS CodeBuild Plugin] Exception in thread "main"
> org.apache.hudi.exception.HoodieException: Failed to run Test Suite
> [AWS CodeBuild Plugin] at
> org.apache.hudi.integ.testsuite.HoodieTestSuiteJob.runTestSuite(HoodieTestSuiteJob.java:208)
> [AWS CodeBuild Plugin] at
> org.apache.hudi.integ.testsuite.HoodieTestSuiteJob.main(HoodieTestSuiteJob.java:170)
> [AWS CodeBuild Plugin] at
> sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> [AWS CodeBuild Plugin] at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> [AWS CodeBuild Plugin] at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> [AWS CodeBuild Plugin] at
> java.lang.reflect.Method.invoke(Method.java:498)
> [AWS CodeBuild Plugin] at
> org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
> [AWS CodeBuild Plugin] at
> org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:845)
> [AWS CodeBuild Plugin] at
> org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:161)
> [AWS CodeBuild Plugin] at
> org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:184)
> [AWS CodeBuild Plugin] at
> org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
> [AWS CodeBuild Plugin] at
> org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:920)
> [AWS CodeBuild Plugin] at
> org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:929)
> [AWS CodeBuild Plugin] at
> org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> [AWS CodeBuild Plugin] Caused by: java.util.concurrent.ExecutionException:
> org.apache.hudi.exception.HoodieException: org.apache.spark.SparkException:
> Task not serializable
> [AWS CodeBuild Plugin] at
> java.util.concurrent.FutureTask.report(FutureTask.java:122)
> [AWS CodeBuild Plugin] at
> java.util.concurrent.FutureTask.get(FutureTask.java:206)
> [AWS CodeBuild Plugin] at
> org.apache.hudi.integ.testsuite.dag.scheduler.DagScheduler.execute(DagScheduler.java:113)
> [AWS CodeBuild Plugin] at
> org.apache.hudi.integ.testsuite.dag.scheduler.DagScheduler.schedule(DagScheduler.java:68)
> [AWS CodeBuild Plugin] at
> org.apache.hudi.integ.testsuite.HoodieTestSuiteJob.runTestSuite(HoodieTestSuiteJob.java:203)
> [AWS CodeBuild Plugin] ... 13 more
> [AWS CodeBuild Plugin] Caused by: org.apache.hudi.exception.HoodieException:
> org.apache.spark.SparkException: Task not serializable
> [AWS CodeBuild Plugin] at
> org.apache.hudi.integ.testsuite.dag.scheduler.DagScheduler.executeNode(DagScheduler.java:146)
> [AWS CodeBuild Plugin] at
> org.apache.hudi.integ.testsuite.dag.scheduler.DagScheduler.lambda$execute$0(DagScheduler.java:105)
> [AWS CodeBuild Plugin] at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> [AWS CodeBuild Plugin] at
> java.util.concurrent.FutureTask.run(FutureTask.java:266)
> [AWS CodeBuild Plugin] at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> [AWS CodeBuild Plugin] at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> [AWS CodeBuild Plugin] at java.lang.Thread.run(Thread.java:748)
> [AWS CodeBuild Plugin] Caused by: org.apache.spark.SparkException: Task not
> serializable
> [AWS CodeBuild Plugin] at
> org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:403)
> [AWS CodeBuild Plugin] at
> org.apache.spark.util.ClosureCleaner$.org$apache$spark$util$ClosureCleaner$$clean(ClosureCleaner.scala:393)
> [AWS CodeBuild Plugin] at
> org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:162)
> [AWS CodeBuild Plugin] at
> org.apache.spark.SparkContext.clean(SparkContext.scala:2326)
> [AWS CodeBuild Plugin] at
> org.apache.spark.rdd.RDD$$anonfun$map$1.apply(RDD.scala:371)
> [AWS CodeBuild Plugin] at
> org.apache.spark.rdd.RDD$$anonfun$map$1.apply(RDD.scala:370)
> [AWS CodeBuild Plugin] at
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> [AWS CodeBuild Plugin] at
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
> [AWS CodeBuild Plugin] at
> org.apache.spark.rdd.RDD.withScope(RDD.scala:363)
> [AWS CodeBuild Plugin] at org.apache.spark.rdd.RDD.map(RDD.scala:370)
> [AWS CodeBuild Plugin] at
> org.apache.spark.api.java.JavaRDDLike$class.map(JavaRDDLike.scala:93)
> [AWS CodeBuild Plugin] at
> org.apache.spark.api.java.AbstractJavaRDDLike.map(JavaRDDLike.scala:45)
> [AWS CodeBuild Plugin] at
> org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter.getNextBatch(HoodieTestSuiteWriter.java:138)
> [AWS CodeBuild Plugin] at
> org.apache.hudi.integ.testsuite.dag.nodes.SparkInsertNode.execute(SparkInsertNode.scala:52)
> [AWS CodeBuild Plugin] at
> org.apache.hudi.integ.testsuite.dag.scheduler.DagScheduler.executeNode(DagScheduler.java:139)
> [AWS CodeBuild Plugin] ... 6 more
> [AWS CodeBuild Plugin] Caused by: java.io.NotSerializableException:
> org.apache.hudi.table.upgrade.SparkUpgradeDowngradeHelper
> [AWS CodeBuild Plugin] Serialization stack:
> [AWS CodeBuild Plugin] - object not serializable (class:
> org.apache.hudi.table.upgrade.SparkUpgradeDowngradeHelper, value:
> org.apache.hudi.table.upgrade.SparkUpgradeDowngradeHelper@77e85a58)
> [AWS CodeBuild Plugin] - field (class:
> org.apache.hudi.client.BaseHoodieWriteClient, name: upgradeDowngradeHelper,
> type: interface org.apache.hudi.table.upgrade.SupportsUpgradeDowngrade)
> [AWS CodeBuild Plugin] - object (class
> org.apache.hudi.client.SparkRDDWriteClient,
> org.apache.hudi.client.SparkRDDWriteClient@50c0b7d2)
> [AWS CodeBuild Plugin] - field (class:
> org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter, name: writeClient,
> type: class org.apache.hudi.client.SparkRDDWriteClient)
> [AWS CodeBuild Plugin] - object (class
> org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter,
> org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter@7b38a6c2)
> [AWS CodeBuild Plugin] - element of array (index: 0)
> [AWS CodeBuild Plugin] - array (class [Ljava.lang.Object;, size 1)
> [AWS CodeBuild Plugin] - field (class:
> java.lang.invoke.SerializedLambda, name: capturedArgs, type: class
> [Ljava.lang.Object;)
> [AWS CodeBuild Plugin] - object (class
> java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=class
> org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter,
> functionalInterfaceMethod=org/apache/spark/api/java/function/Function.call:(Ljava/lang/Object;)Ljava/lang/Object;,
> implementation=invokeSpecial
> org/apache/hudi/integ/testsuite/HoodieTestSuiteWriter.lambda$getNextBatch$c5590ce6$1:(Lorg/apache/hudi/common/model/HoodieRecord;)Lorg/apache/avro/generic/GenericRecord;,
>
> instantiatedMethodType=(Lorg/apache/hudi/common/model/HoodieRecord;)Lorg/apache/avro/generic/GenericRecord;,
> numCaptured=1])
> [AWS CodeBuild Plugin] - writeReplace data (class:
> java.lang.invoke.SerializedLambda)
> [AWS CodeBuild Plugin] - object (class
> org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter$$Lambda$138/1591629766,
> org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter$$Lambda$138/1591629766@18e13131)
> [AWS CodeBuild Plugin] - field (class:
> org.apache.spark.api.java.JavaPairRDD$$anonfun$toScalaFunction$1, name:
> fun$1, type: interface org.apache.spark.api.java.function.Function)
> [AWS CodeBuild Plugin] - object (class
> org.apache.spark.api.java.JavaPairRDD$$anonfun$toScalaFunction$1, <function1>)
> [AWS CodeBuild Plugin] at
> org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:40)
> [AWS CodeBuild Plugin] at
> org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:46)
> [AWS CodeBuild Plugin] at
> org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:100)
> [AWS CodeBuild Plugin] at
> org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:400)
> [AWS CodeBuild Plugin] ... 20 more
> [AWS CodeBuild Plugin]
> [AWS CodeBuild Plugin] 2022/03/09 03:00:06 Command did not exit successfully
> sh run-intig-test-v2.sh HudiIntegTestsManualTest-23-2022-03-09 MERGE_ON_READ
> cow-spark-simple.yaml test.properties
> hudi-integ-test-bundle-0.11.0-SNAPSHOT.jar exit status 1
> [AWS CodeBuild Plugin] 2022/03/09 03:00:06 Phase complete: BUILD State: FAILED
> [AWS CodeBuild Plugin] 2022/03/09 03:00:06 Phase context status code:
> COMMAND_EXECUTION_ERROR Message: Error while executing command: sh
> run-intig-test-v2.sh HudiIntegTestsManualTest-23-2022-03-09 MERGE_ON_READ
> cow-spark-simple.yaml test.properties
> hudi-integ-test-bundle-0.11.0-SNAPSHOT.jar. Reason: exit status 1
> [AWS CodeBuild Plugin] 2022/03/09 03:00:06 Entering phase POST_BUILD
> [AWS CodeBuild Plugin] 2022/03/09 03:00:06 Phase complete: POST_BUILD State:
> SUCCEEDED
> [AWS CodeBuild Plugin] 2022/03/09 03:00:06 Phase context status code:
> Message: {code}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
