voonhous opened a new issue, #14177:
URL: https://github.com/apache/hudi/issues/14177
### Bug Description
**What happened:**
When trying to perform dynamic filtering using MDT files on Trino, the error
below is thrown:
```
io.trino.testing.QueryFailedException: Failed to generate splits for
tests.hudi_multi_fg_pt_v8_mor
at
io.trino.testing.AbstractTestingTrinoClient.execute(AbstractTestingTrinoClient.java:138)
at
io.trino.testing.DistributedQueryRunner.executeInternal(DistributedQueryRunner.java:565)
at
io.trino.testing.DistributedQueryRunner.execute(DistributedQueryRunner.java:548)
at
io.trino.plugin.hudi.TestHudiSmokeTest.testDynamicFilterEnabled_withPartitionPruningUsingDynamicFilter(TestHudiSmokeTest.java:974)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at
java.base/java.util.concurrent.ForkJoinTask.doExec$$$capture(ForkJoinTask.java:507)
at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java)
at
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1460)
at
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:2036)
at
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:189)
Suppressed: java.lang.Exception: SQL: EXPLAIN ANALYZE SELECT t1.id,
t1.name, t1.price, t1.ts, t1.country FROM HUDI_MULTI_FG_PT_V8_MOR t1 INNER JOIN
HUDI_MULTI_FG_PT_V8_MOR t2 ON t1.country = t2.country WHERE t2.price <= 102
at
io.trino.testing.DistributedQueryRunner.executeInternal(DistributedQueryRunner.java:572)
... 8 more
Caused by: io.trino.spi.TrinoException: Failed to generate splits for
tests.hudi_multi_fg_pt_v8_mor
at
io.trino.plugin.hudi.HudiSplitSource.lambda$new$1(HudiSplitSource.java:136)
at
io.trino.plugin.hudi.split.HudiBackgroundSplitLoader.run(HudiBackgroundSplitLoader.java:138)
at
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at
java.base/java.util.concurrent.FutureTask.run$$$capture(FutureTask.java:317)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java)
at
java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
Caused by: org.apache.hudi.exception.HoodieException: Error occurs when
executing map
at
org.apache.hudi.common.function.FunctionWrapper.lambda$throwingMapWrapper$0(FunctionWrapper.java:40)
at
org.apache.hudi.common.data.HoodieListData.lambda$flatMap$1(HoodieListData.java:166)
at
java.base/java.util.stream.ReferencePipeline$7$1FlatMap.accept(ReferencePipeline.java:289)
at
java.base/java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1709)
at
java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:570)
at
java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:560)
at
java.base/java.util.stream.ReduceOps$ReduceTask.doLeaf(ReduceOps.java:960)
at
java.base/java.util.stream.ReduceOps$ReduceTask.doLeaf(ReduceOps.java:934)
at
java.base/java.util.stream.AbstractTask.compute(AbstractTask.java:327)
at
java.base/java.util.concurrent.CountedCompleter.exec(CountedCompleter.java:759)
at
java.base/java.util.concurrent.ForkJoinTask.doExec$$$capture(ForkJoinTask.java:507)
at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java)
at
java.base/java.util.concurrent.ForkJoinTask.invoke(ForkJoinTask.java:676)
at
java.base/java.util.stream.ReduceOps$ReduceOp.evaluateParallel(ReduceOps.java:927)
at
java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:264)
at
java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:727)
at
org.apache.hudi.common.data.HoodieBaseListData.collectAsList(HoodieBaseListData.java:82)
at
org.apache.hudi.common.data.HoodieListData.collectAsList(HoodieListData.java:253)
at
io.trino.plugin.hudi.query.index.HudiPartitionStatsIndexSupport.prunePartitions(HudiPartitionStatsIndexSupport.java:80)
at
io.trino.plugin.hudi.split.HudiBackgroundSplitLoader.getPartitionInfos(HudiBackgroundSplitLoader.java:227)
at
io.trino.plugin.hudi.split.HudiBackgroundSplitLoader.generateSplits(HudiBackgroundSplitLoader.java:145)
at
io.trino.plugin.hudi.split.HudiBackgroundSplitLoader.run(HudiBackgroundSplitLoader.java:130)
... 7 more
Caused by: org.apache.hudi.exception.HoodieException: Exception when reading
log file
at
org.apache.hudi.common.table.log.BaseHoodieLogRecordReader.scanInternalV1(BaseHoodieLogRecordReader.java:393)
at
org.apache.hudi.common.table.log.BaseHoodieLogRecordReader.scanInternal(BaseHoodieLogRecordReader.java:203)
at
org.apache.hudi.common.table.log.HoodieMergedLogRecordReader.performScan(HoodieMergedLogRecordReader.java:100)
at
org.apache.hudi.common.table.log.HoodieMergedLogRecordReader.<init>(HoodieMergedLogRecordReader.java:75)
at
org.apache.hudi.common.table.log.HoodieMergedLogRecordReader.<init>(HoodieMergedLogRecordReader.java:55)
at
org.apache.hudi.common.table.log.HoodieMergedLogRecordReader$Builder.build(HoodieMergedLogRecordReader.java:276)
at
org.apache.hudi.common.table.read.buffer.LogScanningRecordBufferLoader.scanLogFiles(LogScanningRecordBufferLoader.java:53)
at
org.apache.hudi.common.table.read.buffer.DefaultFileGroupRecordBufferLoader.getRecordBuffer(DefaultFileGroupRecordBufferLoader.java:81)
at
org.apache.hudi.common.table.read.HoodieFileGroupReader.initRecordIterators(HoodieFileGroupReader.java:134)
at
org.apache.hudi.common.table.read.HoodieFileGroupReader.getBufferedRecordIterator(HoodieFileGroupReader.java:291)
at
org.apache.hudi.common.table.read.HoodieFileGroupReader.getClosableIterator(HoodieFileGroupReader.java:300)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.readSliceWithFilter(HoodieBackedTableMetadata.java:558)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.readSliceAndFilterByKeysIntoList(HoodieBackedTableMetadata.java:628)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$getRecordsByKeyPrefixes$54ebcbfb$1(HoodieBackedTableMetadata.java:253)
at
org.apache.hudi.common.function.FunctionWrapper.lambda$throwingMapWrapper$0(FunctionWrapper.java:38)
... 28 more
Caused by: org.apache.hudi.exception.HoodieIOException: Failed to deser
records from log files
at
org.apache.hudi.common.table.read.buffer.FileGroupRecordBuffer.getRecordsIterator(FileGroupRecordBuffer.java:207)
at
org.apache.hudi.common.table.read.buffer.KeyBasedFileGroupRecordBuffer.processDataBlock(KeyBasedFileGroupRecordBuffer.java:76)
at
org.apache.hudi.common.table.log.BaseHoodieLogRecordReader.processQueuedBlocksForInstant(BaseHoodieLogRecordReader.java:733)
at
org.apache.hudi.common.table.log.BaseHoodieLogRecordReader.scanInternalV1(BaseHoodieLogRecordReader.java:381)
... 42 more
Caused by: java.io.IOException: Invalid magic: expected TRABLK"$, got
"doc":"A
at
org.apache.hudi.io.hfile.HFileBlockType.readAndCheckMagic(HFileBlockType.java:167)
at
org.apache.hudi.io.hfile.HFileTrailer.deserialize(HFileTrailer.java:118)
at
org.apache.hudi.io.hfile.HFileReaderImpl.readTrailer(HFileReaderImpl.java:302)
at
org.apache.hudi.io.hfile.HFileReaderImpl.initializeMetadata(HFileReaderImpl.java:74)
at
org.apache.hudi.io.hfile.HFileReaderImpl.seekTo(HFileReaderImpl.java:201)
at
org.apache.hudi.io.storage.HoodieNativeAvroHFileReader$RecordByKeyPrefixIterator.<init>(HoodieNativeAvroHFileReader.java:501)
at
org.apache.hudi.io.storage.HoodieNativeAvroHFileReader.getEngineRecordsByKeyPrefixIterator(HoodieNativeAvroHFileReader.java:239)
at
org.apache.hudi.common.table.log.block.HoodieHFileDataBlock.lookupEngineRecords(HoodieHFileDataBlock.java:202)
at
org.apache.hudi.common.table.log.block.HoodieDataBlock.getEngineRecordIterator(HoodieDataBlock.java:266)
at
org.apache.hudi.common.table.read.buffer.FileGroupRecordBuffer.getRecordsIterator(FileGroupRecordBuffer.java:199)
... 45 more
```
**What you expected:**
No error to be thrown.
**Steps to reproduce:**
Run the test using Hudi `1.2.0-SNAPSHOT`
UniqueId:
```
[engine:junit-jupiter]/[class:io.trino.plugin.hudi.TestHudiSmokeTest]/[test-template:testDynamicFilterEnabled_withPartitionPruningUsingDynamicFilter(io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer$TestingTable)]/[test-template-invocation:#2]
```
### Environment
**Hudi version:** 1.2.0-SNAPSHOT
**Query engine:** Trino
**Relevant configs:**
### Logs and Stack Trace
_No response_
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
