This is an automated email from the ASF dual-hosted git repository. junhao pushed a commit to branch release-0.8 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 947e93f687293365d2e081a074853440aa84fd6d Author: Jingsong Lee <[email protected]> AuthorDate: Thu May 30 14:01:21 2024 +0800 [parquet] Pass conf to parquet writer (#3435) --- .../parquet/writer/RowDataParquetBuilder.java | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/RowDataParquetBuilder.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/RowDataParquetBuilder.java index e4874f579..a55766194 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/RowDataParquetBuilder.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/RowDataParquetBuilder.java @@ -22,6 +22,7 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.options.Options; import org.apache.paimon.types.RowType; +import org.apache.hadoop.conf.Configuration; import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.hadoop.ParquetOutputFormat; import org.apache.parquet.hadoop.ParquetWriter; @@ -36,31 +37,31 @@ import java.io.IOException; public class RowDataParquetBuilder implements ParquetBuilder<InternalRow> { private final RowType rowType; - private final Options conf; + private final Configuration conf; - public RowDataParquetBuilder(RowType rowType, Options conf) { + public RowDataParquetBuilder(RowType rowType, Options options) { this.rowType = rowType; - this.conf = conf; + this.conf = new Configuration(false); + options.toMap().forEach(conf::set); } @Override public ParquetWriter<InternalRow> createWriter(OutputFile out, String compression) throws IOException { - return new ParquetRowDataBuilder(out, rowType) + .withConf(conf) .withCompressionCodec(CompressionCodecName.fromConf(getCompression(compression))) .withRowGroupSize( conf.getLong( ParquetOutputFormat.BLOCK_SIZE, ParquetWriter.DEFAULT_BLOCK_SIZE)) .withPageSize( - conf.getInteger( - ParquetOutputFormat.PAGE_SIZE, ParquetWriter.DEFAULT_PAGE_SIZE)) + conf.getInt(ParquetOutputFormat.PAGE_SIZE, ParquetWriter.DEFAULT_PAGE_SIZE)) .withDictionaryPageSize( - conf.getInteger( + conf.getInt( ParquetOutputFormat.DICTIONARY_PAGE_SIZE, ParquetProperties.DEFAULT_DICTIONARY_PAGE_SIZE)) .withMaxPaddingSize( - conf.getInteger( + conf.getInt( ParquetOutputFormat.MAX_PADDING_BYTES, ParquetWriter.MAX_PADDING_SIZE_DEFAULT)) .withDictionaryEncoding( @@ -70,7 +71,7 @@ public class RowDataParquetBuilder implements ParquetBuilder<InternalRow> { .withValidation(conf.getBoolean(ParquetOutputFormat.VALIDATION, false)) .withWriterVersion( ParquetProperties.WriterVersion.fromString( - conf.getString( + conf.get( ParquetOutputFormat.WRITER_VERSION, ParquetProperties.DEFAULT_WRITER_VERSION.toString()))) .build(); @@ -82,8 +83,7 @@ public class RowDataParquetBuilder implements ParquetBuilder<InternalRow> { compressName = compression; } else { compressName = - conf.getString( - ParquetOutputFormat.COMPRESSION, CompressionCodecName.SNAPPY.name()); + conf.get(ParquetOutputFormat.COMPRESSION, CompressionCodecName.SNAPPY.name()); } return compressName; }
