[
https://issues.apache.org/jira/browse/HUDI-5188?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Alexey Kudinkin updated HUDI-5188:
----------------------------------
Fix Version/s: 0.13.0
> ClassCastException: class HoodieBackedTableMetadataWriter$DirectoryInfo
> cannot be cast to class HoodieBackedTableMetadataWriter$DirectoryInfo
> ---------------------------------------------------------------------------------------------------------------------------------------------
>
> Key: HUDI-5188
> URL: https://issues.apache.org/jira/browse/HUDI-5188
> Project: Apache Hudi
> Issue Type: Bug
> Reporter: Alexey Kudinkin
> Priority: Major
> Fix For: 0.13.0
>
>
> After XXX, started to get following exception when running trivial Spark DS
> workloads:
> {code:java}
> import org.apache.hudi.DataSourceWriteOptions
> import org.apache.hudi.DataSourceWriteOptions.{PRECOMBINE_FIELD,
> RECORDKEY_FIELD}
> import org.apache.spark.sql.{SaveMode, SparkSession}
> val spark: SparkSession
> import spark.implicits._
> val basePath = "/tmp/test"
> val writerOpts: Map[String, String] = Map(
> "hoodie.table.name" -> "test",
> "hoodie.table.type" -> "COPY_ON_WRITE",
> PRECOMBINE_FIELD.key() -> "id",
> RECORDKEY_FIELD.key() -> "id",
> DataSourceWriteOptions.PARTITIONPATH_FIELD.key() -> "dt"
> )
> val firstBatchDF = (for (i <- 0 until 10) yield (i, s"a$i", 10 + i, 10000,
> s"2021/03/0${i % 2 + 1}", s"${i % 3}")).toDF("id", "name", "price",
> "version", "dt", "hh")
> firstBatchDF.write.
> options(writerOpts).
> option("hoodie.parquet.compression.codec", "gzip").
> format("hudi").
> mode(SaveMode.Overwrite).
> save(basePath) {code}
> {code:java}
> java.lang.ClassCastException: class
> org.apache.hudi.metadata.HoodieBackedTableMetadataWriter$DirectoryInfo cannot
> be cast to class
> org.apache.hudi.metadata.HoodieBackedTableMetadataWriter$DirectoryInfo
> (org.apache.hudi.metadata.HoodieBackedTableMetadataWriter$DirectoryInfo is in
> unnamed module of loader org.apache.spark.util.ChildFirstURLClassLoader
> @3b1895e;
> org.apache.hudi.metadata.HoodieBackedTableMetadataWriter$DirectoryInfo is in
> unnamed module of loader
> scala.reflect.internal.util.ScalaClassLoader$URLClassLoader @58882a93) at
> org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.listAllPartitions(HoodieBackedTableMetadataWriter.java:645)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.initialCommit(HoodieBackedTableMetadataWriter.java:1070)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.initializeFromFilesystem(HoodieBackedTableMetadataWriter.java:560)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.initializeIfNeeded(HoodieBackedTableMetadataWriter.java:393)
> at
> org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter.initialize(SparkHoodieBackedTableMetadataWriter.java:120)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.<init>(HoodieBackedTableMetadataWriter.java:173)
> at
> org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter.<init>(SparkHoodieBackedTableMetadataWriter.java:89)
> at
> org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter.create(SparkHoodieBackedTableMetadataWriter.java:75)
> at
> org.apache.hudi.client.SparkRDDWriteClient.initializeMetadataTable(SparkRDDWriteClient.java:514)
> at
> org.apache.hudi.client.SparkRDDWriteClient.doInitTable(SparkRDDWriteClient.java:499)
> at
> org.apache.hudi.client.BaseHoodieWriteClient.initTable(BaseHoodieWriteClient.java:1548)
> at
> org.apache.hudi.client.BaseHoodieWriteClient.initTable(BaseHoodieWriteClient.java:1580)
> at
> org.apache.hudi.client.SparkRDDWriteClient.upsert(SparkRDDWriteClient.java:156)
> at
> org.apache.hudi.DataSourceUtils.doWriteOperation(DataSourceUtils.java:206)
> at
> org.apache.hudi.HoodieSparkSqlWriter$.write(HoodieSparkSqlWriter.scala:331)
> at org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:148) at
> org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:45)
> at
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:75)
> at
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:73)
> at
> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:84)
> at
> org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:110)
> at
> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)
> at
> org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)
> at
> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)
> at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775)
> at
> org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
> at
> org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:110)
> at
> org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:106)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:481)
> at
> org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:82)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:481)
> at
> org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:30)
> at
> org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:267)
> at
> org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:263)
> at
> org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:30)
> at
> org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:30)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:457)
> at
> org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:106)
> at
> org.apache.spark.sql.execution.QueryExecution.commandExecuted$lzycompute(QueryExecution.scala:93)
> at
> org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:91)
> at
> org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:128)
> at
> org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:848)
> at
> org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:382)
> at
> org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:355)
> at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
> ... 54 elided
>
> {code}
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
