yuqi1129 opened a new issue, #8372:
URL: https://github.com/apache/gravitino/issues/8372
### Version
main branch
### Describe what's wrong
Caused by: org.apache.spark.SparkException: Job aborted due to stage
failure: Task 0 in stage 0.0 failed 1 times, most recent failure: Lost task 0.0
in stage 0.0 (TID 0) (192.168.0.109 executor driver):
java.io.InvalidObjectException: ReflectiveOperationException during
deserialization
at
java.base/java.lang.invoke.SerializedLambda.readResolve(SerializedLambda.java:280)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at
java.base/java.io.ObjectStreamClass.invokeReadResolve(ObjectStreamClass.java:1190)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2266)
### Error message and/or stacktrace
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File
"/Users/yuqi/venv-spark-3.1/lib/python3.9/site-packages/pyspark/sql/readwriter.py",
line 1372, in csv
self._jwrite.csv(path)
File
"/Users/yuqi/venv-spark-3.1/lib/python3.9/site-packages/py4j/java_gateway.py",
line 1304, in __call__
return_value = get_return_value(
File
"/Users/yuqi/venv-spark-3.1/lib/python3.9/site-packages/pyspark/sql/utils.py",
line 111, in deco
return f(*a, **kw)
File
"/Users/yuqi/venv-spark-3.1/lib/python3.9/site-packages/py4j/protocol.py", line
326, in get_return_value
raise Py4JJavaError(
py4j.protocol.Py4JJavaError: An error occurred while calling o83.csv.
: org.apache.spark.SparkException: Job aborted.
at
org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:231)
at
org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:188)
at
org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108)
at
org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106)
at
org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:131)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at
org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176)
at
org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:132)
at
org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:131)
at
org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:989)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)
at
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775)
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
at
org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:989)
at
org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:438)
at
org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:415)
at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:293)
at org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:979)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:238)
at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: org.apache.spark.SparkException: Job aborted due to stage
failure: Task 0 in stage 0.0 failed 1 times, most recent failure: Lost task 0.0
in stage 0.0 (TID 0) (192.168.0.109 executor driver):
java.io.InvalidObjectException: ReflectiveOperationException during
deserialization
at
java.base/java.lang.invoke.SerializedLambda.readResolve(SerializedLambda.java:280)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at
java.base/java.io.ObjectStreamClass.invokeReadResolve(ObjectStreamClass.java:1190)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2266)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream.readArray(ObjectInputStream.java:2157)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1721)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:509)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:467)
at
scala.collection.immutable.List$SerializationProxy.readObject(List.scala:488)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at
java.base/java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1100)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2423)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:509)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:467)
at
scala.collection.immutable.List$SerializationProxy.readObject(List.scala:488)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at
java.base/java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1100)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2423)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:509)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:467)
at
scala.collection.immutable.List$SerializationProxy.readObject(List.scala:488)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at
java.base/java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1100)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2423)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:509)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:467)
at
scala.collection.immutable.List$SerializationProxy.readObject(List.scala:488)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at
java.base/java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1100)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2423)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:509)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:467)
at
org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:76)
at
org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:115)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:83)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at
org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:498)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:501)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.reflect.InvocationTargetException
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at
java.base/java.lang.invoke.SerializedLambda.readResolve(SerializedLambda.java:278)
... 109 more
Caused by: java.lang.IllegalArgumentException: too many arguments
at
java.base/java.lang.invoke.LambdaMetafactory.altMetafactory(LambdaMetafactory.java:511)
at
scala.runtime.LambdaDeserializer$.makeCallSite$1(LambdaDeserializer.scala:105)
at
scala.runtime.LambdaDeserializer$.deserializeLambda(LambdaDeserializer.scala:114)
at
scala.runtime.LambdaDeserialize.deserializeLambda(LambdaDeserialize.java:38)
at
org.apache.spark.api.python.SerDeUtil$.$deserializeLambda$(SerDeUtil.scala)
... 114 more
Driver stacktrace:
at
org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2303)
at
org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2252)
at
org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2251)
at
scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at
scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at
org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2251)
at
org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1124)
at
org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1124)
at scala.Option.foreach(Option.scala:407)
at
org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1124)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2490)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2432)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2421)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
at
org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:902)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196)
at
org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:200)
... 33 more
Caused by: java.io.InvalidObjectException: ReflectiveOperationException
during deserialization
at
java.base/java.lang.invoke.SerializedLambda.readResolve(SerializedLambda.java:280)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at
java.base/java.io.ObjectStreamClass.invokeReadResolve(ObjectStreamClass.java:1190)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2266)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream.readArray(ObjectInputStream.java:2157)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1721)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:509)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:467)
at
scala.collection.immutable.List$SerializationProxy.readObject(List.scala:488)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at
java.base/java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1100)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2423)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:509)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:467)
at
scala.collection.immutable.List$SerializationProxy.readObject(List.scala:488)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at
java.base/java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1100)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2423)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:509)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:467)
at
scala.collection.immutable.List$SerializationProxy.readObject(List.scala:488)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at
java.base/java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1100)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2423)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:509)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:467)
at
scala.collection.immutable.List$SerializationProxy.readObject(List.scala:488)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at
java.base/java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1100)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2423)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream$FieldValues.<init>(ObjectInputStream.java:2606)
at
java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2457)
at
java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2257)
at
java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1733)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:509)
at
java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:467)
at
org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:76)
at
org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:115)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:83)
at org.apache.spark.scheduler.Task.run(Task.scala:131)
at
org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:498)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:501)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: java.lang.reflect.InvocationTargetException
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at
java.base/java.lang.invoke.SerializedLambda.readResolve(SerializedLambda.java:278)
... 109 more
Caused by: java.lang.IllegalArgumentException: too many arguments
at
java.base/java.lang.invoke.LambdaMetafactory.altMetafactory(LambdaMetafactory.java:511)
at
scala.runtime.LambdaDeserializer$.makeCallSite$1(LambdaDeserializer.scala:105)
at
scala.runtime.LambdaDeserializer$.deserializeLambda(LambdaDeserializer.scala:114)
at
scala.runtime.LambdaDeserialize.deserializeLambda(LambdaDeserialize.java:38)
at
org.apache.spark.api.python.SerDeUtil$.$deserializeLambda$(SerDeUtil.scala)
... 114 more
### How to reproduce
1. Build with command `./gradlew release `./gradlew release -x test -x rat`
2. Then run it with JDK17
```
import logging
logging.basicConfig(level=logging.INFO)
from gravitino import NameIdentifier, GravitinoClient, Catalog, Fileset,
GravitinoAdminClient
gravitino_url = "http://localhost:8090";
metalake_name = "test"
catalog_name = "oss_catalog"
schema_name = "schema"
fileset_name = "fileset01"
fileset_ident = NameIdentifier.of(schema_name, fileset_name)
gravitino_admin_client = GravitinoAdminClient(uri=gravitino_url)
gravitino_client = GravitinoClient(uri=gravitino_url,
metalake_name=metalake_name)
from pyspark.sql import SparkSession
import os
os.environ["PYSPARK_SUBMIT_ARGS"] = "--jars
/Users/yuqi/project/graviton/bundles/aliyun/build/libs/gravitino-aliyun-1.0.0-SNAPSHOT.jar,/Users/yuqi/project/graviton/clients/filesystem-hadoop3-runtime/build/libs/gravitino-filesystem-hadoop3-runtime-1.0.0-SNAPSHOT.jar,/Users/yuqi/Downloads/hadoop-jars/aliyun-sdk-oss-2.8.3.jar,/Users/yuqi/Downloads/hadoop-jars/hadoop-aliyun-3.2.0.jar,/Users/yuqi/Downloads/hadoop-jars/jdom-1.1.jar
--conf
\"spark.driver.extraJavaOptions=--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005\" --conf
\"spark.executor.extraJavaOptions=--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006\" --master
local[1] pyspark-shell"
spark = SparkSession.builder \
.appName("s3_fielset_test") \
.config("spark.hadoop.fs.AbstractFileSystem.gvfs.impl",
"org.apache.gravitino.filesystem.hadoop.Gvfs") \
.config("spark.hadoop.fs.gvfs.impl",
"org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystem") \
.config("spark.hadoop.fs.gravitino.server.uri", "http://localhost:8090";)
\
.config("spark.hadoop.fs.gravitino.client.metalake", "test") \
.config("spark.hadoop.oss-access-key-id", "xxx") \
.config("spark.hadoop.oss-secret-access-key", "xxx") \
.config("spark.hadoop.oss-endpoint",
"https://oss-cn-shanghai.aliyuncs.com";) \
.config("spark.driver.memory", "2g") \
.config("spark.driver.port", "2048") \
.getOrCreate()
data = [("Alice", 25), ("Bob", 30), ("Cathy", 45)]
columns = ["Name", "Age"]
spark_df = spark.createDataFrame(data, schema=columns)
gvfs_path =
f"gvfs://fileset/{catalog_name}/{schema_name}/{fileset_name}/people"
spark_df.coalesce(1).write \
.mode("overwrite") \
.option("header", "true") \
.csv(gvfs_path)
```
### Additional context
_No response_
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
