clp007 commented on issue #11446:
URL: https://github.com/apache/hudi/issues/11446#issuecomment-2522254520
I had the same problem when I deleted the data of some partitions ，The data
uses secondary partition, event_date=2024-12-03/event_name=active ,about 200
partitions every day
Environment Description
Hudi version : 0.15.0
Spark version : 3.5.1
Hive version : NA
Hadoop version : 3.3.6
Storage (HDFS/S3/GCS..) : GCS
24/12/06 06:22:39 WARN HttpParser: URI is too large >8192
24/12/06 06:22:39 ERROR PriorityBasedFileSystemView: Got error running
preferred function. Trying secondary
org.apache.hudi.exception.HoodieRemoteException: URI Too Long
at
org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.loadPartitions(RemoteHoodieTableFileSystemView.java:447)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.loadPartitions(RemoteHoodieTableFileSystemView.java:465)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.common.table.view.PriorityBasedFileSystemView.lambda$loadPartitions$6e5c444d$1(PriorityBasedFileSystemView.java:187)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.common.table.view.PriorityBasedFileSystemView.execute(PriorityBasedFileSystemView.java:69)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.common.table.view.PriorityBasedFileSystemView.loadPartitions(PriorityBasedFileSystemView.java:185)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.table.action.clean.CleanPlanActionExecutor.requestClean(CleanPlanActionExecutor.java:133)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.table.action.clean.CleanPlanActionExecutor.requestClean(CleanPlanActionExecutor.java:174)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.table.action.clean.CleanPlanActionExecutor.execute(CleanPlanActionExecutor.java:200)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.table.HoodieSparkCopyOnWriteTable.scheduleCleaning(HoodieSparkCopyOnWriteTable.java:212)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.client.BaseHoodieTableServiceClient.scheduleTableServiceInternal(BaseHoodieTableServiceClient.java:647)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.client.BaseHoodieTableServiceClient.clean(BaseHoodieTableServiceClient.java:746)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.client.BaseHoodieWriteClient.clean(BaseHoodieWriteClient.java:843)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.client.BaseHoodieWriteClient.clean(BaseHoodieWriteClient.java:816)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.client.BaseHoodieWriteClient.clean(BaseHoodieWriteClient.java:847)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.client.BaseHoodieWriteClient.autoCleanOnCommit(BaseHoodieWriteClient.java:581)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.client.BaseHoodieWriteClient.mayBeCleanAndArchive(BaseHoodieWriteClient.java:560)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.client.BaseHoodieWriteClient.commitStats(BaseHoodieWriteClient.java:251)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.client.SparkRDDWriteClient.commit(SparkRDDWriteClient.java:108)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.HoodieSparkSqlWriterInternal.commitAndPerformPostOperations(HoodieSparkSqlWriter.scala:1082)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.HoodieSparkSqlWriterInternal.writeInternal(HoodieSparkSqlWriter.scala:508)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.HoodieSparkSqlWriterInternal.write(HoodieSparkSqlWriter.scala:187)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.HoodieSparkSqlWriter$.write(HoodieSparkSqlWriter.scala:125)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:168)
~[hudi-spark3.5-bundle_2.12-0.15.0.jar:0.15.0]
at
org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:48)
~[spark-sql_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:75)
~[spark-sql_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:73)
~[spark-sql_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:84)
~[spark-sql_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:107)
~[spark-sql_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$6(SQLExecution.scala:125)
~[spark-sql_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:201)
~[spark-sql_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:108)
~[spark-sql_2.12-3.5.1.jar:3.5.1]
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:900)
~[spark-sql_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:66)
~[spark-sql_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:107)
~[spark-sql_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:98)
~[spark-sql_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:473)
~[spark-catalyst_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:76)
[spark-sql-api_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:473)
[spark-catalyst_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:32)
[spark-catalyst_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:267)
[spark-catalyst_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:263)
[spark-catalyst_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:32)
[spark-catalyst_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:32)
[spark-catalyst_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:449)
[spark-catalyst_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:98)
[spark-sql_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.execution.QueryExecution.commandExecuted$lzycompute(QueryExecution.scala:85)
[spark-sql_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:83)
[spark-sql_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:142)
[spark-sql_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:859)
[spark-sql_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:388)
[spark-sql_2.12-3.5.1.jar:3.5.1]
at
org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:361)
[spark-sql_2.12-3.5.1.jar:3.5.1]
at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:240)
[spark-sql_2.12-3.5.1.jar:3.5.1]
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
~[?:?]
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
~[?:?]
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:?]
at java.base/java.lang.reflect.Method.invoke(Method.java:566) ~[?:?]
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
[py4j-0.10.9.7.jar:?]
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
[py4j-0.10.9.7.jar:?]
at py4j.Gateway.invoke(Gateway.java:282) [py4j-0.10.9.7.jar:?]
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
[py4j-0.10.9.7.jar:?]
at py4j.commands.CallCommand.execute(CallCommand.java:79)
[py4j-0.10.9.7.jar:?]
at
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
[py4j-0.10.9.7.jar:?]
at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
[py4j-0.10.9.7.jar:?]
at java.base/java.lang.Thread.run(Thread.java:829) [?:?]
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
