zaza commented on issue #6900:
URL: https://github.com/apache/hudi/issues/6900#issuecomment-1649736303
This is definitely still an issue, we were hit by an error that looks
identical to what @umehrot2 reported a while ago:
```
ERROR UpsertPartitioner: Error trying to compute average bytes/record
org.apache.hudi.exception.HoodieIOException: Could not read commit details
from
s3://tasktop-data-platform-dev-analytical-data/simulator/workstreams/.hoodie/20230714152804208.commit
at
org.apache.hudi.common.table.timeline.HoodieActiveTimeline.readDataFromPath(HoodieActiveTimeline.java:824)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.common.table.timeline.HoodieActiveTimeline.getInstantDetails(HoodieActiveTimeline.java:310)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.common.table.timeline.HoodieDefaultTimeline.getInstantDetails(HoodieDefaultTimeline.java:438)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.table.action.commit.UpsertPartitioner.averageBytesPerRecord(UpsertPartitioner.java:380)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.table.action.commit.UpsertPartitioner.assignInserts(UpsertPartitioner.java:169)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.table.action.commit.UpsertPartitioner.<init>(UpsertPartitioner.java:98)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.table.action.commit.BaseSparkCommitActionExecutor.getUpsertPartitioner(BaseSparkCommitActionExecutor.java:404)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.table.action.commit.BaseSparkCommitActionExecutor.getPartitioner(BaseSparkCommitActionExecutor.java:224)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.table.action.commit.BaseSparkCommitActionExecutor.execute(BaseSparkCommitActionExecutor.java:170)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.table.action.commit.BaseSparkCommitActionExecutor.execute(BaseSparkCommitActionExecutor.java:83)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.table.action.commit.BaseWriteHelper.write(BaseWriteHelper.java:68)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.table.action.commit.SparkUpsertCommitActionExecutor.execute(SparkUpsertCommitActionExecutor.java:44)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.table.HoodieSparkCopyOnWriteTable.upsert(HoodieSparkCopyOnWriteTable.java:107)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.table.HoodieSparkCopyOnWriteTable.upsert(HoodieSparkCopyOnWriteTable.java:96)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.client.SparkRDDWriteClient.upsert(SparkRDDWriteClient.java:140)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.DataSourceUtils.doWriteOperation(DataSourceUtils.java:214)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.HoodieSparkSqlWriter$.write(HoodieSparkSqlWriter.scala:372)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:150)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:47)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:75)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:73)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:84)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:104)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:107)
~[spark-catalyst_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.SQLExecution$.withTracker(SQLExecution.scala:224)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.SQLExecution$.executeQuery$1(SQLExecution.scala:114)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$7(SQLExecution.scala:139)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:107)
~[spark-catalyst_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.SQLExecution$.withTracker(SQLExecution.scala:224)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$6(SQLExecution.scala:139)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:245)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:138)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:779)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:68)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:101)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:97)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:626)
~[spark-catalyst_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:179)
~[spark-catalyst_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:626)
~[spark-catalyst_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:31)
~[spark-catalyst_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:267)
~[spark-catalyst_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:263)
~[spark-catalyst_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:31)
~[spark-catalyst_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:31)
~[spark-catalyst_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:602)
~[spark-catalyst_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:97)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.QueryExecution.commandExecuted$lzycompute(QueryExecution.scala:84)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:82)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:125)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:860)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:390)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:363)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:239)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
com.planview.ep.dap.workstreams.WorkStreamsTableWriter.write(WorkStreamsTableWriter.java:67)
~[__app__.jar:?]
at
com.planview.ep.dap.workstreams.java.StreamingJob.lambda$start$89fce565$1(StreamingJob.java:80)
~[__app__.jar:?]
at
org.apache.spark.sql.streaming.DataStreamWriter.$anonfun$foreachBatch$1(DataStreamWriter.scala:493)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.streaming.DataStreamWriter.$anonfun$foreachBatch$1$adapted(DataStreamWriter.scala:493)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.streaming.sources.ForeachBatchSink.addBatch(ForeachBatchSink.scala:32)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$runBatch$17(MicroBatchExecution.scala:665)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:107)
~[spark-catalyst_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.SQLExecution$.withTracker(SQLExecution.scala:224)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.SQLExecution$.executeQuery$1(SQLExecution.scala:114)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$7(SQLExecution.scala:139)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:107)
~[spark-catalyst_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.SQLExecution$.withTracker(SQLExecution.scala:224)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$6(SQLExecution.scala:139)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:245)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:138)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:779)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:68)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$runBatch$16(MicroBatchExecution.scala:663)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken(ProgressReporter.scala:375)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken$(ProgressReporter.scala:373)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:68)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.streaming.MicroBatchExecution.runBatch(MicroBatchExecution.scala:663)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$runActivatedStream$2(MicroBatchExecution.scala:256)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
~[scala-library-2.12.15.jar:?]
at
org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken(ProgressReporter.scala:375)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken$(ProgressReporter.scala:373)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:68)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.streaming.MicroBatchExecution.$anonfun$runActivatedStream$1(MicroBatchExecution.scala:219)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.streaming.ProcessingTimeExecutor.execute(TriggerExecutor.scala:67)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.streaming.MicroBatchExecution.runActivatedStream(MicroBatchExecution.scala:213)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.streaming.StreamExecution.$anonfun$runStream$1(StreamExecution.scala:307)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
~[scala-library-2.12.15.jar:?]
at
org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:779)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:285)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
at
org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:208)
~[spark-sql_2.12-3.3.2-amzn-0.jar:3.3.2-amzn-0]
Caused by: java.io.FileNotFoundException: No such file or directory
's3://tasktop-data-platform-dev-analytical-data/simulator/workstreams/.hoodie/20230714152804208.commit'
at
com.amazon.ws.emr.hadoop.fs.s3n.S3NativeFileSystem.getFileStatus(S3NativeFileSystem.java:529)
~[emrfs-hadoop-assembly-2.56.0.jar:?]
at
com.amazon.ws.emr.hadoop.fs.s3n.S3NativeFileSystem.open(S3NativeFileSystem.java:940)
~[emrfs-hadoop-assembly-2.56.0.jar:?]
at
com.amazon.ws.emr.hadoop.fs.s3n.S3NativeFileSystem.open(S3NativeFileSystem.java:932)
~[emrfs-hadoop-assembly-2.56.0.jar:?]
at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:983)
~[hadoop-client-api-3.3.3-amzn-3.jar:?]
at
com.amazon.ws.emr.hadoop.fs.EmrFileSystem.open(EmrFileSystem.java:197)
~[emrfs-hadoop-assembly-2.56.0.jar:?]
at
org.apache.hudi.common.fs.HoodieWrapperFileSystem.open(HoodieWrapperFileSystem.java:476)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
at
org.apache.hudi.common.table.timeline.HoodieActiveTimeline.readDataFromPath(HoodieActiveTimeline.java:821)
~[org.apache.hudi_hudi-spark3.3-bundle_2.12-0.13.1.jar:0.13.1]
```
We're on EMR 6.11.0
What additional information do you need in order to troubleshoot this? Are
there any configuration settings that would help us mitigate the problem?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
