[
https://issues.apache.org/jira/browse/HUDI-2835?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Shawn updated HUDI-2835:
------------------------
Description:
flink: 1.13.3
hudi: 0.10.0-SNAPSHOT
hive: 2.1.1
I found the following exception when testing HoodieFlinkStreamer。
I tracked the code and found that the reason is that the default value given by
the Flink WRITE_PARQUET_PAGE_SIZE parameter is 1, but the
CapacityByteArrayOutputStream constructor requires no less than the default
value of 64。
Why the default value of WRITE_PARQUET_PAGE_SIZE is 1, can I fix it?
{code:java}
java.lang.IllegalArgumentException: maxCapacityHint can't be less than
initialSlabSize 64 1
at org.apache.parquet.Preconditions.checkArgument(Preconditions.java:53)
at
org.apache.parquet.bytes.CapacityByteArrayOutputStream.<init>(CapacityByteArrayOutputStream.java:153)
at
org.apache.parquet.column.values.rle.RunLengthBitPackingHybridEncoder.<init>(RunLengthBitPackingHybridEncoder.java:125)
at
org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter.<init>(RunLengthBitPackingHybridValuesWriter.java:34)
at
org.apache.parquet.column.ParquetProperties.newColumnDescriptorValuesWriter(ParquetProperties.java:131)
at
org.apache.parquet.column.ParquetProperties.newDefinitionLevelWriter(ParquetProperties.java:123)
at
org.apache.parquet.column.impl.ColumnWriterV1.createDLWriter(ColumnWriterV1.java:47)
at
org.apache.parquet.column.impl.ColumnWriterBase.<init>(ColumnWriterBase.java:65)
at
org.apache.parquet.column.impl.ColumnWriterV1.<init>(ColumnWriterV1.java:37)
at
org.apache.parquet.column.impl.ColumnWriteStoreV1.createColumnWriter(ColumnWriteStoreV1.java:41)
at
org.apache.parquet.column.impl.ColumnWriteStoreBase.<init>(ColumnWriteStoreBase.java:94)
at
org.apache.parquet.column.impl.ColumnWriteStoreV1.<init>(ColumnWriteStoreV1.java:30)
at
org.apache.parquet.column.ParquetProperties.newColumnWriteStore(ParquetProperties.java:180)
at
org.apache.parquet.hadoop.InternalParquetRecordWriter.initStore(InternalParquetRecordWriter.java:106)
at
org.apache.parquet.hadoop.InternalParquetRecordWriter.<init>(InternalParquetRecordWriter.java:96)
at org.apache.parquet.hadoop.ParquetWriter.<init>(ParquetWriter.java:292)
at org.apache.parquet.hadoop.ParquetWriter.<init>(ParquetWriter.java:227)
at
org.apache.hudi.io.storage.HoodieParquetWriter.<init>(HoodieParquetWriter.java:56)
at
org.apache.hudi.io.storage.HoodieFileWriterFactory.newParquetFileWriter(HoodieFileWriterFactory.java:76)
at
org.apache.hudi.io.storage.HoodieFileWriterFactory.newParquetFileWriter(HoodieFileWriterFactory.java:63)
at
org.apache.hudi.io.storage.HoodieFileWriterFactory.getFileWriter(HoodieFileWriterFactory.java:49)
at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:101)
at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:67)
at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:115)
at
org.apache.hudi.table.HoodieFlinkCopyOnWriteTable.handleInsert(HoodieFlinkCopyOnWriteTable.java:390)
at
org.apache.hudi.table.action.compact.HoodieCompactor.compact(HoodieCompactor.java:201)
at
org.apache.hudi.sink.compact.CompactFunction.doCompaction(CompactFunction.java:106)
at
org.apache.hudi.sink.compact.CompactFunction.lambda$processElement$0(CompactFunction.java:94)
at
org.apache.hudi.sink.utils.NonThrownExecutor.lambda$execute$0(NonThrownExecutor.java:93)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748) {code}
was:
flink: 1.13.3
hudi: 0.10.0-SNAPSHOT
hive: 2.1.1
I tracked the code and found that the reason is that the default value given by
the Flink WRITE_PARQUET_PAGE_SIZE parameter is 1, but the
CapacityByteArrayOutputStream constructor requires no less than the default
value of 64。
Why the default value of WRITE_PARQUET_PAGE_SIZE is 1, can I fix it?
{code:java}
java.lang.IllegalArgumentException: maxCapacityHint can't be less than
initialSlabSize 64 1
at org.apache.parquet.Preconditions.checkArgument(Preconditions.java:53)
at
org.apache.parquet.bytes.CapacityByteArrayOutputStream.<init>(CapacityByteArrayOutputStream.java:153)
at
org.apache.parquet.column.values.rle.RunLengthBitPackingHybridEncoder.<init>(RunLengthBitPackingHybridEncoder.java:125)
at
org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter.<init>(RunLengthBitPackingHybridValuesWriter.java:34)
at
org.apache.parquet.column.ParquetProperties.newColumnDescriptorValuesWriter(ParquetProperties.java:131)
at
org.apache.parquet.column.ParquetProperties.newDefinitionLevelWriter(ParquetProperties.java:123)
at
org.apache.parquet.column.impl.ColumnWriterV1.createDLWriter(ColumnWriterV1.java:47)
at
org.apache.parquet.column.impl.ColumnWriterBase.<init>(ColumnWriterBase.java:65)
at
org.apache.parquet.column.impl.ColumnWriterV1.<init>(ColumnWriterV1.java:37)
at
org.apache.parquet.column.impl.ColumnWriteStoreV1.createColumnWriter(ColumnWriteStoreV1.java:41)
at
org.apache.parquet.column.impl.ColumnWriteStoreBase.<init>(ColumnWriteStoreBase.java:94)
at
org.apache.parquet.column.impl.ColumnWriteStoreV1.<init>(ColumnWriteStoreV1.java:30)
at
org.apache.parquet.column.ParquetProperties.newColumnWriteStore(ParquetProperties.java:180)
at
org.apache.parquet.hadoop.InternalParquetRecordWriter.initStore(InternalParquetRecordWriter.java:106)
at
org.apache.parquet.hadoop.InternalParquetRecordWriter.<init>(InternalParquetRecordWriter.java:96)
at org.apache.parquet.hadoop.ParquetWriter.<init>(ParquetWriter.java:292)
at org.apache.parquet.hadoop.ParquetWriter.<init>(ParquetWriter.java:227)
at
org.apache.hudi.io.storage.HoodieParquetWriter.<init>(HoodieParquetWriter.java:56)
at
org.apache.hudi.io.storage.HoodieFileWriterFactory.newParquetFileWriter(HoodieFileWriterFactory.java:76)
at
org.apache.hudi.io.storage.HoodieFileWriterFactory.newParquetFileWriter(HoodieFileWriterFactory.java:63)
at
org.apache.hudi.io.storage.HoodieFileWriterFactory.getFileWriter(HoodieFileWriterFactory.java:49)
at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:101)
at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:67)
at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:115)
at
org.apache.hudi.table.HoodieFlinkCopyOnWriteTable.handleInsert(HoodieFlinkCopyOnWriteTable.java:390)
at
org.apache.hudi.table.action.compact.HoodieCompactor.compact(HoodieCompactor.java:201)
at
org.apache.hudi.sink.compact.CompactFunction.doCompaction(CompactFunction.java:106)
at
org.apache.hudi.sink.compact.CompactFunction.lambda$processElement$0(CompactFunction.java:94)
at
org.apache.hudi.sink.utils.NonThrownExecutor.lambda$execute$0(NonThrownExecutor.java:93)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748) {code}
> maxCapacityHint can't be less than initialSlabSize 64 1
> -------------------------------------------------------
>
> Key: HUDI-2835
> URL: https://issues.apache.org/jira/browse/HUDI-2835
> Project: Apache Hudi
> Issue Type: Bug
> Components: Flink Integration
> Affects Versions: 0.10.0
> Reporter: Shawn
> Priority: Major
>
> flink: 1.13.3
> hudi: 0.10.0-SNAPSHOT
> hive: 2.1.1
> I found the following exception when testing HoodieFlinkStreamer。
> I tracked the code and found that the reason is that the default value given
> by the Flink WRITE_PARQUET_PAGE_SIZE parameter is 1, but the
> CapacityByteArrayOutputStream constructor requires no less than the default
> value of 64。
> Why the default value of WRITE_PARQUET_PAGE_SIZE is 1, can I fix it?
> {code:java}
> java.lang.IllegalArgumentException: maxCapacityHint can't be less than
> initialSlabSize 64 1
> at org.apache.parquet.Preconditions.checkArgument(Preconditions.java:53)
> at
> org.apache.parquet.bytes.CapacityByteArrayOutputStream.<init>(CapacityByteArrayOutputStream.java:153)
> at
> org.apache.parquet.column.values.rle.RunLengthBitPackingHybridEncoder.<init>(RunLengthBitPackingHybridEncoder.java:125)
> at
> org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter.<init>(RunLengthBitPackingHybridValuesWriter.java:34)
> at
> org.apache.parquet.column.ParquetProperties.newColumnDescriptorValuesWriter(ParquetProperties.java:131)
> at
> org.apache.parquet.column.ParquetProperties.newDefinitionLevelWriter(ParquetProperties.java:123)
> at
> org.apache.parquet.column.impl.ColumnWriterV1.createDLWriter(ColumnWriterV1.java:47)
> at
> org.apache.parquet.column.impl.ColumnWriterBase.<init>(ColumnWriterBase.java:65)
> at
> org.apache.parquet.column.impl.ColumnWriterV1.<init>(ColumnWriterV1.java:37)
> at
> org.apache.parquet.column.impl.ColumnWriteStoreV1.createColumnWriter(ColumnWriteStoreV1.java:41)
> at
> org.apache.parquet.column.impl.ColumnWriteStoreBase.<init>(ColumnWriteStoreBase.java:94)
> at
> org.apache.parquet.column.impl.ColumnWriteStoreV1.<init>(ColumnWriteStoreV1.java:30)
> at
> org.apache.parquet.column.ParquetProperties.newColumnWriteStore(ParquetProperties.java:180)
> at
> org.apache.parquet.hadoop.InternalParquetRecordWriter.initStore(InternalParquetRecordWriter.java:106)
> at
> org.apache.parquet.hadoop.InternalParquetRecordWriter.<init>(InternalParquetRecordWriter.java:96)
> at org.apache.parquet.hadoop.ParquetWriter.<init>(ParquetWriter.java:292)
> at org.apache.parquet.hadoop.ParquetWriter.<init>(ParquetWriter.java:227)
> at
> org.apache.hudi.io.storage.HoodieParquetWriter.<init>(HoodieParquetWriter.java:56)
> at
> org.apache.hudi.io.storage.HoodieFileWriterFactory.newParquetFileWriter(HoodieFileWriterFactory.java:76)
> at
> org.apache.hudi.io.storage.HoodieFileWriterFactory.newParquetFileWriter(HoodieFileWriterFactory.java:63)
> at
> org.apache.hudi.io.storage.HoodieFileWriterFactory.getFileWriter(HoodieFileWriterFactory.java:49)
> at
> org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:101)
> at
> org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:67)
> at
> org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:115)
> at
> org.apache.hudi.table.HoodieFlinkCopyOnWriteTable.handleInsert(HoodieFlinkCopyOnWriteTable.java:390)
> at
> org.apache.hudi.table.action.compact.HoodieCompactor.compact(HoodieCompactor.java:201)
> at
> org.apache.hudi.sink.compact.CompactFunction.doCompaction(CompactFunction.java:106)
> at
> org.apache.hudi.sink.compact.CompactFunction.lambda$processElement$0(CompactFunction.java:94)
> at
> org.apache.hudi.sink.utils.NonThrownExecutor.lambda$execute$0(NonThrownExecutor.java:93)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748) {code}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
