(paimon) branch master updated: [parquet] Pass conf to parquet writer (#3435)

[junhao]

This is an automated email from the ASF dual-hosted git repository.

junhao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 4cbb443cc [parquet] Pass conf to parquet writer (#3435)
4cbb443cc is described below

commit 4cbb443cc194db7b83629701684436cd856975bc
Author: Jingsong Lee <jingsongl...@gmail.com>
AuthorDate: Thu May 30 14:01:21 2024 +0800

    [parquet] Pass conf to parquet writer (#3435)
---
 .../parquet/writer/RowDataParquetBuilder.java      | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git 
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/RowDataParquetBuilder.java
 
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/RowDataParquetBuilder.java
index e4874f579..a55766194 100644
--- 
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/RowDataParquetBuilder.java
+++ 
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/RowDataParquetBuilder.java
@@ -22,6 +22,7 @@ import org.apache.paimon.data.InternalRow;
 import org.apache.paimon.options.Options;
 import org.apache.paimon.types.RowType;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.parquet.column.ParquetProperties;
 import org.apache.parquet.hadoop.ParquetOutputFormat;
 import org.apache.parquet.hadoop.ParquetWriter;
@@ -36,31 +37,31 @@ import java.io.IOException;
 public class RowDataParquetBuilder implements ParquetBuilder<InternalRow> {
 
     private final RowType rowType;
-    private final Options conf;
+    private final Configuration conf;
 
-    public RowDataParquetBuilder(RowType rowType, Options conf) {
+    public RowDataParquetBuilder(RowType rowType, Options options) {
         this.rowType = rowType;
-        this.conf = conf;
+        this.conf = new Configuration(false);
+        options.toMap().forEach(conf::set);
     }
 
     @Override
     public ParquetWriter<InternalRow> createWriter(OutputFile out, String 
compression)
             throws IOException {
-
         return new ParquetRowDataBuilder(out, rowType)
+                .withConf(conf)
                 
.withCompressionCodec(CompressionCodecName.fromConf(getCompression(compression)))
                 .withRowGroupSize(
                         conf.getLong(
                                 ParquetOutputFormat.BLOCK_SIZE, 
ParquetWriter.DEFAULT_BLOCK_SIZE))
                 .withPageSize(
-                        conf.getInteger(
-                                ParquetOutputFormat.PAGE_SIZE, 
ParquetWriter.DEFAULT_PAGE_SIZE))
+                        conf.getInt(ParquetOutputFormat.PAGE_SIZE, 
ParquetWriter.DEFAULT_PAGE_SIZE))
                 .withDictionaryPageSize(
-                        conf.getInteger(
+                        conf.getInt(
                                 ParquetOutputFormat.DICTIONARY_PAGE_SIZE,
                                 
ParquetProperties.DEFAULT_DICTIONARY_PAGE_SIZE))
                 .withMaxPaddingSize(
-                        conf.getInteger(
+                        conf.getInt(
                                 ParquetOutputFormat.MAX_PADDING_BYTES,
                                 ParquetWriter.MAX_PADDING_SIZE_DEFAULT))
                 .withDictionaryEncoding(
@@ -70,7 +71,7 @@ public class RowDataParquetBuilder implements 
ParquetBuilder<InternalRow> {
                 
.withValidation(conf.getBoolean(ParquetOutputFormat.VALIDATION, false))
                 .withWriterVersion(
                         ParquetProperties.WriterVersion.fromString(
-                                conf.getString(
+                                conf.get(
                                         ParquetOutputFormat.WRITER_VERSION,
                                         
ParquetProperties.DEFAULT_WRITER_VERSION.toString())))
                 .build();
@@ -82,8 +83,7 @@ public class RowDataParquetBuilder implements 
ParquetBuilder<InternalRow> {
             compressName = compression;
         } else {
             compressName =
-                    conf.getString(
-                            ParquetOutputFormat.COMPRESSION, 
CompressionCodecName.SNAPPY.name());
+                    conf.get(ParquetOutputFormat.COMPRESSION, 
CompressionCodecName.SNAPPY.name());
         }
         return compressName;
     }