[parquet-mr] branch master updated: PARQUET-1615: getRecordWriter shouldn't hardcode CREAT mode when new ParquetFileWriter (#660)

[gabor]

This is an automated email from the ASF dual-hosted git repository.

gabor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git


The following commit(s) were added to refs/heads/master by this push:
     new 8ff867a  PARQUET-1615: getRecordWriter shouldn't hardcode CREAT mode 
when new ParquetFileWriter (#660)
8ff867a is described below

commit 8ff867a2e183f50b7b2f2c6e51d07c5314577ce0
Author: Alan Jin <lantao...@users.noreply.github.com>
AuthorDate: Tue Jul 9 16:27:48 2019 +0800

    PARQUET-1615: getRecordWriter shouldn't hardcode CREAT mode when new 
ParquetFileWriter (#660)
---
 .../apache/parquet/hadoop/ParquetOutputFormat.java  | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git 
a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java
 
b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java
index 04cbd15..cd25b23 100644
--- 
a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java
+++ 
b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java
@@ -366,21 +366,36 @@ public class ParquetOutputFormat<T> extends 
FileOutputFormat<Void, T> {
   @Override
   public RecordWriter<Void, T> getRecordWriter(TaskAttemptContext 
taskAttemptContext)
       throws IOException, InterruptedException {
+    return getRecordWriter(taskAttemptContext, Mode.CREATE);
+  }
+
+  public RecordWriter<Void, T> getRecordWriter(TaskAttemptContext 
taskAttemptContext, Mode mode)
+      throws IOException, InterruptedException {
 
     final Configuration conf = getConfiguration(taskAttemptContext);
 
     CompressionCodecName codec = getCodec(taskAttemptContext);
     String extension = codec.getExtension() + ".parquet";
     Path file = getDefaultWorkFile(taskAttemptContext, extension);
-    return getRecordWriter(conf, file, codec);
+    return getRecordWriter(conf, file, codec, mode);
   }
 
   public RecordWriter<Void, T> getRecordWriter(TaskAttemptContext 
taskAttemptContext, Path file)
+    throws IOException, InterruptedException {
+    return getRecordWriter(taskAttemptContext, file, Mode.CREATE);
+  }
+
+  public RecordWriter<Void, T> getRecordWriter(TaskAttemptContext 
taskAttemptContext, Path file, Mode mode)
       throws IOException, InterruptedException {
-    return getRecordWriter(getConfiguration(taskAttemptContext), file, 
getCodec(taskAttemptContext));
+    return getRecordWriter(getConfiguration(taskAttemptContext), file, 
getCodec(taskAttemptContext), mode);
   }
 
   public RecordWriter<Void, T> getRecordWriter(Configuration conf, Path file, 
CompressionCodecName codec)
+      throws IOException, InterruptedException {
+    return getRecordWriter(conf, file, codec, Mode.CREATE);
+  }
+
+  public RecordWriter<Void, T> getRecordWriter(Configuration conf, Path file, 
CompressionCodecName codec, Mode mode)
         throws IOException, InterruptedException {
     final WriteSupport<T> writeSupport = getWriteSupport(conf);
 
@@ -417,7 +432,7 @@ public class ParquetOutputFormat<T> extends 
FileOutputFormat<Void, T> {
 
     WriteContext init = writeSupport.init(conf);
     ParquetFileWriter w = new 
ParquetFileWriter(HadoopOutputFile.fromPath(file, conf),
-        init.getSchema(), Mode.CREATE, blockSize, maxPaddingSize, 
props.getColumnIndexTruncateLength());
+        init.getSchema(), mode, blockSize, maxPaddingSize, 
props.getColumnIndexTruncateLength());
     w.start();
 
     float maxLoad = conf.getFloat(ParquetOutputFormat.MEMORY_POOL_RATIO,