This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new efbbb67 [HUDI-2241] Explicit parallelism for flink bulk insert (#3357)
efbbb67 is described below
commit efbbb67420b7082a3960f3d32215dabd959f5525
Author: Danny Chan <[email protected]>
AuthorDate: Thu Jul 29 09:57:37 2021 +0800
[HUDI-2241] Explicit parallelism for flink bulk insert (#3357)
---
.../src/main/java/org/apache/hudi/table/HoodieTableSink.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git
a/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSink.java
b/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSink.java
index 37d1068..8f3ddbc 100644
--- a/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSink.java
+++ b/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSink.java
@@ -89,12 +89,19 @@ public class HoodieTableSink implements DynamicTableSink,
SupportsPartitioning,
final String writeOperation = this.conf.get(FlinkOptions.OPERATION);
if (WriteOperationType.fromValue(writeOperation) ==
WriteOperationType.BULK_INSERT) {
this.conf.set(FlinkOptions.WRITE_BULK_INSERT_PARTITION_SORTED,
this.supportsGrouping);
+ if (this.supportsGrouping) {
+ // if partition grouping is true, the input records would be sorted
by the partition
+ // path, we need to chain the SORT operator and writer operator to
keep the record
+ // sequence
+ dataStream.getTransformation()
+ .setParallelism(conf.getInteger(FlinkOptions.WRITE_TASKS));
+ }
BulkInsertWriteOperator.OperatorFactory<RowData> operatorFactory =
BulkInsertWriteOperator.getFactory(this.conf, rowType);
return dataStream.transform("hoodie_bulk_insert_write",
TypeInformation.of(Object.class),
operatorFactory)
// follow the parallelism of upstream operators to avoid shuffle
- .setParallelism(dataStream.getParallelism())
+ .setParallelism(conf.getInteger(FlinkOptions.WRITE_TASKS))
.addSink(new CleanFunction<>(conf))
.setParallelism(1)
.name("clean_commits");