This is an automated email from the ASF dual-hosted git repository.

danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new efbbb67  [HUDI-2241] Explicit parallelism for flink bulk insert (#3357)
efbbb67 is described below

commit efbbb67420b7082a3960f3d32215dabd959f5525
Author: Danny Chan <[email protected]>
AuthorDate: Thu Jul 29 09:57:37 2021 +0800

    [HUDI-2241] Explicit parallelism for flink bulk insert (#3357)
---
 .../src/main/java/org/apache/hudi/table/HoodieTableSink.java     | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git 
a/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSink.java 
b/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSink.java
index 37d1068..8f3ddbc 100644
--- a/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSink.java
+++ b/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSink.java
@@ -89,12 +89,19 @@ public class HoodieTableSink implements DynamicTableSink, 
SupportsPartitioning,
       final String writeOperation = this.conf.get(FlinkOptions.OPERATION);
       if (WriteOperationType.fromValue(writeOperation) == 
WriteOperationType.BULK_INSERT) {
         this.conf.set(FlinkOptions.WRITE_BULK_INSERT_PARTITION_SORTED, 
this.supportsGrouping);
+        if (this.supportsGrouping) {
+          // if partition grouping is true, the input records would be sorted 
by the partition
+          // path, we need to chain the SORT operator and writer operator to 
keep the record
+          // sequence
+          dataStream.getTransformation()
+              .setParallelism(conf.getInteger(FlinkOptions.WRITE_TASKS));
+        }
         BulkInsertWriteOperator.OperatorFactory<RowData> operatorFactory = 
BulkInsertWriteOperator.getFactory(this.conf, rowType);
         return dataStream.transform("hoodie_bulk_insert_write",
             TypeInformation.of(Object.class),
             operatorFactory)
             // follow the parallelism of upstream operators to avoid shuffle
-            .setParallelism(dataStream.getParallelism())
+            .setParallelism(conf.getInteger(FlinkOptions.WRITE_TASKS))
             .addSink(new CleanFunction<>(conf))
             .setParallelism(1)
             .name("clean_commits");

Reply via email to