[
https://issues.apache.org/jira/browse/HUDI-6578?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ethan Guo updated HUDI-6578:
----------------------------
Description:
When parquet is used as the log file data block format, the following exception
is thrown:
{code:java}
org.apache.hudi.exception.HoodieException: Exception when reading log file
at
org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.scanInternalV2(AbstractHoodieLogRecordReader.java:586)
~[classes/:?]
at
org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.scanInternal(AbstractHoodieLogRecordReader.java:218)
~[classes/:?]
at
org.apache.hudi.common.table.log.HoodieUnMergedLogRecordScanner.scan(HoodieUnMergedLogRecordScanner.java:61)
~[classes/:?]
at
org.apache.hudi.table.action.compact.plan.generators.HoodieLogCompactionPlanGenerator.isFileSliceEligibleForLogCompaction(HoodieLogCompactionPlanGenerator.java:102)
~[classes/:?]
at
org.apache.hudi.table.action.compact.plan.generators.HoodieLogCompactionPlanGenerator.filterFileSlice(HoodieLogCompactionPlanGenerator.java:70)
~[classes/:?]
at
org.apache.hudi.table.action.compact.plan.generators.BaseHoodieCompactionPlanGenerator.lambda$null$2(BaseHoodieCompactionPlanGenerator.java:119)
~[classes/:?]
at
java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:174)
~[?:1.8.0_292]
at
java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
~[?:1.8.0_292]
at java.util.ArrayList$ArrayListSpliterator.tryAdvance(ArrayList.java:1361)
~[?:1.8.0_292]
at
java.util.stream.StreamSpliterators$WrappingSpliterator.lambda$initPartialTraversalState$0(StreamSpliterators.java:295)
~[?:1.8.0_292]
at
java.util.stream.StreamSpliterators$AbstractWrappingSpliterator.fillBuffer(StreamSpliterators.java:207)
~[?:1.8.0_292]
at
java.util.stream.StreamSpliterators$AbstractWrappingSpliterator.doAdvance(StreamSpliterators.java:162)
~[?:1.8.0_292]
at
java.util.stream.StreamSpliterators$WrappingSpliterator.tryAdvance(StreamSpliterators.java:301)
~[?:1.8.0_292]
at java.util.Spliterators$1Adapter.hasNext(Spliterators.java:681)
~[?:1.8.0_292]
at
scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:45)
~[scala-library-2.12.15.jar:?]
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:490)
~[scala-library-2.12.15.jar:?]
at scala.collection.Iterator.foreach(Iterator.scala:943)
~[scala-library-2.12.15.jar:?]
at scala.collection.Iterator.foreach$(Iterator.scala:943)
~[scala-library-2.12.15.jar:?]
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
~[scala-library-2.12.15.jar:?]
at scala.collection.generic.Growable.$plus$plus$eq(Growable.scala:62)
~[scala-library-2.12.15.jar:?]
at scala.collection.generic.Growable.$plus$plus$eq$(Growable.scala:53)
~[scala-library-2.12.15.jar:?]
at
scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:105)
~[scala-library-2.12.15.jar:?]
at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:49)
~[scala-library-2.12.15.jar:?]
at scala.collection.TraversableOnce.to(TraversableOnce.scala:366)
~[scala-library-2.12.15.jar:?]
at scala.collection.TraversableOnce.to$(TraversableOnce.scala:364)
~[scala-library-2.12.15.jar:?]
at scala.collection.AbstractIterator.to(Iterator.scala:1431)
~[scala-library-2.12.15.jar:?]
at scala.collection.TraversableOnce.toBuffer(TraversableOnce.scala:358)
~[scala-library-2.12.15.jar:?]
at scala.collection.TraversableOnce.toBuffer$(TraversableOnce.scala:358)
~[scala-library-2.12.15.jar:?]
at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1431)
~[scala-library-2.12.15.jar:?]
at scala.collection.TraversableOnce.toArray(TraversableOnce.scala:345)
~[scala-library-2.12.15.jar:?]
at scala.collection.TraversableOnce.toArray$(TraversableOnce.scala:339)
~[scala-library-2.12.15.jar:?]
at scala.collection.AbstractIterator.toArray(Iterator.scala:1431)
~[scala-library-2.12.15.jar:?]
at org.apache.spark.rdd.RDD.$anonfun$collect$2(RDD.scala:1021)
~[spark-core_2.12-3.3.1.jar:3.3.1]
at org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2268)
~[spark-core_2.12-3.3.1.jar:3.3.1]
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
~[spark-core_2.12-3.3.1.jar:3.3.1]
at org.apache.spark.scheduler.Task.run(Task.scala:136)
~[spark-core_2.12-3.3.1.jar:3.3.1]
at
org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:548)
~[spark-core_2.12-3.3.1.jar:3.3.1]
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1504)
~[spark-core_2.12-3.3.1.jar:3.3.1]
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:551)
~[spark-core_2.12-3.3.1.jar:3.3.1]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
~[?:1.8.0_292]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
~[?:1.8.0_292]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_292]
Caused by: java.lang.UnsupportedOperationException: Block type not yet
supported.
at
org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.scanInternalV2(AbstractHoodieLogRecordReader.java:507)
~[classes/:?]
... 41 more {code}
> Fix log compaction on parquet data block in log files
> -----------------------------------------------------
>
> Key: HUDI-6578
> URL: https://issues.apache.org/jira/browse/HUDI-6578
> Project: Apache Hudi
> Issue Type: Bug
> Reporter: Ethan Guo
> Assignee: Ethan Guo
> Priority: Major
> Fix For: 0.14.0
>
>
> When parquet is used as the log file data block format, the following
> exception is thrown:
> {code:java}
> org.apache.hudi.exception.HoodieException: Exception when reading log file
> at
> org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.scanInternalV2(AbstractHoodieLogRecordReader.java:586)
> ~[classes/:?]
> at
> org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.scanInternal(AbstractHoodieLogRecordReader.java:218)
> ~[classes/:?]
> at
> org.apache.hudi.common.table.log.HoodieUnMergedLogRecordScanner.scan(HoodieUnMergedLogRecordScanner.java:61)
> ~[classes/:?]
> at
> org.apache.hudi.table.action.compact.plan.generators.HoodieLogCompactionPlanGenerator.isFileSliceEligibleForLogCompaction(HoodieLogCompactionPlanGenerator.java:102)
> ~[classes/:?]
> at
> org.apache.hudi.table.action.compact.plan.generators.HoodieLogCompactionPlanGenerator.filterFileSlice(HoodieLogCompactionPlanGenerator.java:70)
> ~[classes/:?]
> at
> org.apache.hudi.table.action.compact.plan.generators.BaseHoodieCompactionPlanGenerator.lambda$null$2(BaseHoodieCompactionPlanGenerator.java:119)
> ~[classes/:?]
> at
> java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:174)
> ~[?:1.8.0_292]
> at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> ~[?:1.8.0_292]
> at
> java.util.ArrayList$ArrayListSpliterator.tryAdvance(ArrayList.java:1361)
> ~[?:1.8.0_292]
> at
> java.util.stream.StreamSpliterators$WrappingSpliterator.lambda$initPartialTraversalState$0(StreamSpliterators.java:295)
> ~[?:1.8.0_292]
> at
> java.util.stream.StreamSpliterators$AbstractWrappingSpliterator.fillBuffer(StreamSpliterators.java:207)
> ~[?:1.8.0_292]
> at
> java.util.stream.StreamSpliterators$AbstractWrappingSpliterator.doAdvance(StreamSpliterators.java:162)
> ~[?:1.8.0_292]
> at
> java.util.stream.StreamSpliterators$WrappingSpliterator.tryAdvance(StreamSpliterators.java:301)
> ~[?:1.8.0_292]
> at java.util.Spliterators$1Adapter.hasNext(Spliterators.java:681)
> ~[?:1.8.0_292]
> at
> scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:45)
> ~[scala-library-2.12.15.jar:?]
> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:490)
> ~[scala-library-2.12.15.jar:?]
> at scala.collection.Iterator.foreach(Iterator.scala:943)
> ~[scala-library-2.12.15.jar:?]
> at scala.collection.Iterator.foreach$(Iterator.scala:943)
> ~[scala-library-2.12.15.jar:?]
> at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
> ~[scala-library-2.12.15.jar:?]
> at scala.collection.generic.Growable.$plus$plus$eq(Growable.scala:62)
> ~[scala-library-2.12.15.jar:?]
> at scala.collection.generic.Growable.$plus$plus$eq$(Growable.scala:53)
> ~[scala-library-2.12.15.jar:?]
> at
> scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:105)
> ~[scala-library-2.12.15.jar:?]
> at
> scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:49)
> ~[scala-library-2.12.15.jar:?]
> at scala.collection.TraversableOnce.to(TraversableOnce.scala:366)
> ~[scala-library-2.12.15.jar:?]
> at scala.collection.TraversableOnce.to$(TraversableOnce.scala:364)
> ~[scala-library-2.12.15.jar:?]
> at scala.collection.AbstractIterator.to(Iterator.scala:1431)
> ~[scala-library-2.12.15.jar:?]
> at scala.collection.TraversableOnce.toBuffer(TraversableOnce.scala:358)
> ~[scala-library-2.12.15.jar:?]
> at scala.collection.TraversableOnce.toBuffer$(TraversableOnce.scala:358)
> ~[scala-library-2.12.15.jar:?]
> at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1431)
> ~[scala-library-2.12.15.jar:?]
> at scala.collection.TraversableOnce.toArray(TraversableOnce.scala:345)
> ~[scala-library-2.12.15.jar:?]
> at scala.collection.TraversableOnce.toArray$(TraversableOnce.scala:339)
> ~[scala-library-2.12.15.jar:?]
> at scala.collection.AbstractIterator.toArray(Iterator.scala:1431)
> ~[scala-library-2.12.15.jar:?]
> at org.apache.spark.rdd.RDD.$anonfun$collect$2(RDD.scala:1021)
> ~[spark-core_2.12-3.3.1.jar:3.3.1]
> at
> org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2268)
> ~[spark-core_2.12-3.3.1.jar:3.3.1]
> at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
> ~[spark-core_2.12-3.3.1.jar:3.3.1]
> at org.apache.spark.scheduler.Task.run(Task.scala:136)
> ~[spark-core_2.12-3.3.1.jar:3.3.1]
> at
> org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:548)
> ~[spark-core_2.12-3.3.1.jar:3.3.1]
> at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1504)
> ~[spark-core_2.12-3.3.1.jar:3.3.1]
> at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:551)
> ~[spark-core_2.12-3.3.1.jar:3.3.1]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> ~[?:1.8.0_292]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> ~[?:1.8.0_292]
> at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_292]
> Caused by: java.lang.UnsupportedOperationException: Block type not yet
> supported.
> at
> org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.scanInternalV2(AbstractHoodieLogRecordReader.java:507)
> ~[classes/:?]
> ... 41 more {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
