Shawn Chang created HUDI-9154:
---------------------------------
Summary: Bootstrap will fail because of not supporting col stats
Key: HUDI-9154
URL: https://issues.apache.org/jira/browse/HUDI-9154
Project: Apache Hudi
Issue Type: Bug
Reporter: Shawn Chang
{code:java}
Exception in thread "main" org.apache.hudi.exception.HoodieException: Updating
data table config to latest set of columns indexed with col stats failed
at
org.apache.hudi.client.HoodieColumnStatsIndexUtils.updateColsToIndex(HoodieColumnStatsIndexUtils.java:76)
at
org.apache.hudi.table.action.commit.BaseCommitActionExecutor.commit(BaseCommitActionExecutor.java:237)
at
org.apache.hudi.table.action.bootstrap.SparkBootstrapCommitActionExecutor.commit(SparkBootstrapCommitActionExecutor.java:218)
at
org.apache.hudi.table.action.commit.BaseCommitActionExecutor.autoCommit(BaseCommitActionExecutor.java:207)
at
org.apache.hudi.table.action.commit.BaseCommitActionExecutor.commitOnAutoCommit(BaseCommitActionExecutor.java:188)
at
org.apache.hudi.table.action.bootstrap.SparkBootstrapCommitActionExecutor.updateIndexAndCommitIfNeeded(SparkBootstrapCommitActionExecutor.java:180)
at
org.apache.hudi.table.action.bootstrap.SparkBootstrapCommitActionExecutor.metadataBootstrap(SparkBootstrapCommitActionExecutor.java:162)
at
org.apache.hudi.table.action.bootstrap.SparkBootstrapCommitActionExecutor.execute(SparkBootstrapCommitActionExecutor.java:127)
at
org.apache.hudi.table.HoodieSparkCopyOnWriteTable.bootstrap(HoodieSparkCopyOnWriteTable.java:199)
at
org.apache.hudi.client.SparkRDDWriteClient.bootstrap(SparkRDDWriteClient.java:123)
at
org.apache.hudi.HoodieSparkSqlWriterInternal.bootstrap(HoodieSparkSqlWriter.scala:786)
at
org.apache.hudi.HoodieSparkSqlWriter$.bootstrap(HoodieSparkSqlWriter.scala:140)
at org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:183)
at
org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:48)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:75)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:73)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:84)
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:126)
at
org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:108)
at
org.apache.spark.sql.execution.SQLExecution$.withTracker(SQLExecution.scala:384)
at
org.apache.spark.sql.execution.SQLExecution$.executeQuery$1(SQLExecution.scala:157)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$10(SQLExecution.scala:220)
at
org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:108)
at
org.apache.spark.sql.execution.SQLExecution$.withTracker(SQLExecution.scala:384)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$9(SQLExecution.scala:220)
at
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:405)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:219)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:901)
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:83)
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:74)
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:123)
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:114)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:520)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:77)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:520)
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:34)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:303)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:299)
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:34)
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:34)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:496)
at
org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:114)
at
org.apache.spark.sql.execution.QueryExecution.commandExecuted$lzycompute(QueryExecution.scala:101)
at
org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:99)
at
org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:164)
at
org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:884)
at
org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:405)
at
org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:365)
at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:244)
at
code.amazonaws.emr.examples.hudi.bootstrap.HudiBootstrapTestCOW$.runBootstrapTest(HudiBootstrapTestCOW.scala:57)
at
code.amazonaws.emr.examples.hudi.bootstrap.HudiBootstrapTestCOW$.main(HudiBootstrapTestCOW.scala:150)
at
code.amazonaws.emr.examples.hudi.bootstrap.HudiBootstrapTestCOW.main(HudiBootstrapTestCOW.scala)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
at
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1112)
at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:200)
at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:223)
at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:92)
at
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1204)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1213)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: org.apache.hudi.exception.HoodieNotSupportedException: col stats is
not supported with bootstrap operation
at
org.apache.hudi.table.action.bootstrap.SparkBootstrapCommitActionExecutor.updateColumnsToIndexForColumnStats(SparkBootstrapCommitActionExecutor.java:225)
at
org.apache.hudi.table.action.commit.BaseCommitActionExecutor.lambda$commit$b950a45b$1(BaseCommitActionExecutor.java:239)
at
org.apache.hudi.client.HoodieColumnStatsIndexUtils.updateColsToIndex(HoodieColumnStatsIndexUtils.java:73)
... 63 more
{code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
