This is an automated email from the ASF dual-hosted git repository. XiaoHongbo-Hope pushed a commit to branch release-2.1 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit f8c9bb2dc8dfd104c254e9fb74a8fa5850436d24 Author: JingsongLi <[email protected]> AuthorDate: Fri Sep 18 13:15:57 2026 +0800 [format] Simplify buffered Avro encoder selection --- .../java/org/apache/paimon/format/FileFormat.java | 9 ++------ .../apache/paimon/format/FileFormatFactory.java | 25 ---------------------- .../apache/paimon/format/avro/AvroFileFormat.java | 4 +--- .../paimon/format/avro/AvroFileFormatTest.java | 4 ++-- 4 files changed, 5 insertions(+), 37 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java index 7a06e544ea..f17c23b255 100644 --- a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java +++ b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java @@ -74,10 +74,6 @@ public abstract class FileFormat { } public static FileFormat fromIdentifier(String identifier, Options options) { - return fromIdentifier(identifier, options, false); - } - - private static FileFormat fromIdentifier(String identifier, Options options, boolean manifest) { return fromIdentifier( normalizeFileFormat(identifier), new FormatContext( @@ -86,8 +82,7 @@ public abstract class FileFormat { options.get(CoreOptions.WRITE_BATCH_SIZE), options.get(CoreOptions.WRITE_BATCH_MEMORY), options.get(CoreOptions.FILE_COMPRESSION_ZSTD_LEVEL), - options.get(CoreOptions.FILE_BLOCK_SIZE), - manifest)); + options.get(CoreOptions.FILE_BLOCK_SIZE))); } /** Create a {@link FileFormat} from format identifier and format options. */ @@ -125,6 +120,6 @@ public abstract class FileFormat { // Manifest blocks retain the Avro default independently of data-file block sizing. Options manifestOptions = new Options(options.toMap()); manifestOptions.remove(CoreOptions.FILE_BLOCK_SIZE); - return FileFormat.fromIdentifier(CoreOptions.FILE_FORMAT_AVRO, manifestOptions, true); + return FileFormat.fromIdentifier(CoreOptions.FILE_FORMAT_AVRO, manifestOptions); } } diff --git a/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java b/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java index 157f151ad2..b5cba150fe 100644 --- a/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java +++ b/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java @@ -41,7 +41,6 @@ public interface FileFormatFactory { private final MemorySize writeBatchMemory; private final int zstdLevel; @Nullable private final MemorySize blockSize; - private final boolean manifest; @VisibleForTesting public FormatContext(Options options, int readBatchSize, int writeBatchSize) { @@ -64,31 +63,12 @@ public interface FileFormatFactory { MemorySize writeBatchMemory, int zstdLevel, @Nullable MemorySize blockSize) { - this( - options, - readBatchSize, - writeBatchSize, - writeBatchMemory, - zstdLevel, - blockSize, - false); - } - - public FormatContext( - Options options, - int readBatchSize, - int writeBatchSize, - MemorySize writeBatchMemory, - int zstdLevel, - @Nullable MemorySize blockSize, - boolean manifest) { this.options = options; this.readBatchSize = readBatchSize; this.writeBatchSize = writeBatchSize; this.writeBatchMemory = writeBatchMemory; this.zstdLevel = zstdLevel; this.blockSize = blockSize; - this.manifest = manifest; } public Options options() { @@ -116,11 +96,6 @@ public interface FileFormatFactory { return blockSize; } - /** Whether the format is used for manifest metadata rather than data files. */ - public boolean isManifest() { - return manifest; - } - public boolean caseSensitive() { return options.getOptional(CatalogOptions.CASE_SENSITIVE).orElse(true); } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java index 845cce65d5..ad243b6132 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java @@ -68,7 +68,6 @@ public class AvroFileFormat extends FileFormat { private final Options options; private final int zstdLevel; @Nullable private final MemorySize blockSize; - private final boolean useBufferedEncoder; public AvroFileFormat(FormatContext context) { super(IDENTIFIER); @@ -76,7 +75,6 @@ public class AvroFileFormat extends FileFormat { this.options = getIdentifierPrefixOptions(context.options()); this.zstdLevel = context.zstdLevel(); this.blockSize = context.blockSize(); - this.useBufferedEncoder = !context.isManifest(); } @Override @@ -160,7 +158,7 @@ public class AvroFileFormat extends FileFormat { @Override public FormatWriter create(PositionOutputStream out, String compression) throws IOException { - return createBlockWriter(out, rowType, compression, useBufferedEncoder); + return createBlockWriter(out, rowType, compression, true); } } } diff --git a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java index 251f3ca974..bef9ef8873 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java @@ -261,7 +261,7 @@ public class AvroFileFormatTest { @ParameterizedTest @ValueSource(strings = {"avro", "parquet", "orc"}) - void testManifestWriterUsesDirectEncoder(String dataFileFormat) throws IOException { + void testManifestRowWriterUsesBufferedEncoder(String dataFileFormat) throws IOException { Options options = new Options(); options.set(CoreOptions.FILE_FORMAT, dataFileFormat); FileFormat manifestFormat = FileFormat.manifestFormat(new CoreOptions(options)); @@ -275,7 +275,7 @@ public class AvroFileFormatTest { assertThat(writer) .extracting("writer") .extracting("bufOut") - .isInstanceOf(DirectBinaryEncoder.class); + .isInstanceOf(BufferedBinaryEncoder.class); writer.addElement(GenericRow.of(42)); } }
