This is an automated email from the ASF dual-hosted git repository. XiaoHongbo-Hope pushed a commit to branch release-2.1 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit f95105959014176e44f1638b44eeb3833b79a29f Author: umi <[email protected]> AuthorDate: Thu Sep 17 11:32:37 2026 +0800 [format] Support file.block-size for Avro (#9901) --- docs/generated/core_configuration.html | 2 +- .../main/java/org/apache/paimon/CoreOptions.java | 3 +- .../apache/paimon/format/avro/AvroFileFormat.java | 6 ++ .../paimon/format/avro/AvroFileFormatTest.java | 71 ++++++++++++++++++++++ 4 files changed, 80 insertions(+), 2 deletions(-) diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index cf48d473c4..19f6d52603 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -738,7 +738,7 @@ under the License. <td><h5>file.block-size</h5></td> <td style="word-wrap: break-word;">(none)</td> <td>MemorySize</td> - <td>File block size of format, default value of orc stripe is 64 MB, and parquet row group is 128 MB.</td> + <td>File block size of format, default value of orc stripe is 64 MB, parquet row group is 128 MB, and avro block is 64000 bytes. For avro, this is the uncompressed block size threshold.</td> </tr> <tr> <td><h5>file.compression</h5></td> diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 9bb324b780..20c59b1f95 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -426,7 +426,8 @@ public class CoreOptions implements Serializable { .memoryType() .noDefaultValue() .withDescription( - "File block size of format, default value of orc stripe is 64 MB, and parquet row group is 128 MB."); + "File block size of format, default value of orc stripe is 64 MB, parquet row group is 128 MB, " + + "and avro block is 64000 bytes. For avro, this is the uncompressed block size threshold."); public static final ConfigOption<MemorySize> FILE_INDEX_IN_MANIFEST_THRESHOLD = key("file-index.in-manifest-threshold") diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java index 69512d5ee7..a23c66af21 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java @@ -29,6 +29,7 @@ import org.apache.paimon.fs.CloseShieldOutputStream; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.options.ConfigOption; import org.apache.paimon.options.ConfigOptions; +import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.statistics.SimpleColStatsCollector; @@ -65,12 +66,14 @@ public class AvroFileFormat extends FileFormat { private final Options options; private final int zstdLevel; + @Nullable private final MemorySize blockSize; public AvroFileFormat(FormatContext context) { super(IDENTIFIER); this.options = getIdentifierPrefixOptions(context.options()); this.zstdLevel = context.zstdLevel(); + this.blockSize = context.blockSize(); } @Override @@ -93,6 +96,9 @@ public class AvroFileFormat extends FileFormat { AvroRowDatumWriter datumWriter = new AvroRowDatumWriter(rowType); DataFileWriter<InternalRow> writer = new DataFileWriter<>(datumWriter); writer.setCodec(createCodecFactory(compression)); + if (blockSize != null) { + writer.setSyncInterval(Math.toIntExact(blockSize.getBytes())); + } writer.setFlushOnEveryBlock(false); writer.create(schema, new CloseShieldOutputStream(out)); return new AvroBlockWriter(writer, out, schema); diff --git a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java index c7ca45b88f..33b5798b5b 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java @@ -48,6 +48,9 @@ import org.apache.avro.io.EncoderFactory; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import java.io.ByteArrayOutputStream; import java.io.File; @@ -76,6 +79,74 @@ public class AvroFileFormatTest { fileFormat = new AvroFileFormat(new FormatContext(new Options(), 1024, 1024)); } + @ParameterizedTest + @CsvSource({ + ", 64000, null", + ", 64000, deflate", + "1 kb, 1024, null", + "1 kb, 1024, deflate", + "4 kb, 4096, null", + "4 kb, 4096, deflate", + "128 kb, 131072, null", + "128 kb, 131072, deflate" + }) + void testFileBlockSize(String blockSize, int expectedBlockSize, String compression) + throws IOException { + Options options = new Options(); + if (blockSize != null) { + options.setString("file.block-size", blockSize); + } + FileFormat format = FileFormat.fromIdentifier("avro", options); + RowType rowType = DataTypes.ROW(DataTypes.INT().notNull()).notNull(); + LocalFileIO fileIO = LocalFileIO.create(); + Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString()); + int numRecords = 300_000; + + try (PositionOutputStream out = fileIO.newOutputStream(file, false); + FormatWriter writer = + format.createWriterFactory(rowType).create(out, compression)) { + // Each record is encoded as one byte, so record counts also give uncompressed sizes. + for (int i = 0; i < numRecords; i++) { + writer.addElement(GenericRow.of(0)); + } + } + + long records = 0; + try (AvroBlockReader reader = new AvroBlockReader(fileIO.newInputStream(file))) { + while (reader.hasNextBlock()) { + AvroRawBlock block = reader.nextBorrowedRawBlock(); + long expectedRecords = Math.min(expectedBlockSize, numRecords - records); + assertThat(block.recordCount()).isEqualTo(expectedRecords); + assertThat(block.decompress(null).remaining()).isEqualTo((int) expectedRecords); + records += block.recordCount(); + } + } + assertThat(records).isEqualTo(numRecords); + } + + @ParameterizedTest + @ValueSource(longs = {2147483648L, 4294968320L, Long.MAX_VALUE}) + void testFileBlockSizeOverflow(long blockSize) throws IOException { + Options options = new Options(); + options.setString("file.block-size", Long.toString(blockSize)); + FileFormat format = FileFormat.fromIdentifier("avro", options); + RowType rowType = DataTypes.ROW(DataTypes.INT().notNull()).notNull(); + LocalFileIO fileIO = LocalFileIO.create(); + Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString()); + + try (PositionOutputStream out = fileIO.newOutputStream(file, false)) { + assertThatThrownBy( + () -> { + try (FormatWriter writer = + format.createWriterFactory(rowType).create(out, "null")) { + writer.addElement(GenericRow.of(0)); + } + }) + .isInstanceOf(ArithmeticException.class) + .hasMessage("integer overflow"); + } + } + @Test public void testSupportedDataTypes() { ArrayList<DataField> dataFields = new ArrayList<>();
