[hudi] branch master updated: [HUDI-6763] Optimize collect calls (#9561)

[danny0405]

This is an automated email from the ASF dual-hosted git repository.

danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 2f7e9caebb0 [HUDI-6763] Optimize collect calls (#9561)
2f7e9caebb0 is described below

commit 2f7e9caebb0e7f68a7cc1a9c541cc67440eafa44
Author: Tim Brown <t...@onehouse.ai>
AuthorDate: Wed Aug 30 20:37:23 2023 -0500

    [HUDI-6763] Optimize collect calls (#9561)
---
 .../table/action/commit/BaseSparkCommitActionExecutor.java | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git 
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
 
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
index 7383f428e0a..040cc798747 100644
--- 
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
+++ 
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
@@ -286,7 +286,9 @@ public abstract class BaseSparkCommitActionExecutor<T> 
extends
 
   @Override
   protected void 
setCommitMetadata(HoodieWriteMetadata<HoodieData<WriteStatus>> result) {
-    
result.setCommitMetadata(Option.of(CommitUtils.buildMetadata(result.getWriteStatuses().map(WriteStatus::getStat).collectAsList(),
+    List<HoodieWriteStat> writeStats = 
result.getWriteStatuses().map(WriteStatus::getStat).collectAsList();
+    result.setWriteStats(writeStats);
+    result.setCommitMetadata(Option.of(CommitUtils.buildMetadata(writeStats,
         result.getPartitionToReplaceFileIds(),
         extraMetadata, operationType, getSchemaToStoreInCommit(), 
getCommitActionType())));
   }
@@ -294,16 +296,14 @@ public abstract class BaseSparkCommitActionExecutor<T> 
extends
   @Override
   protected void commit(Option<Map<String, String>> extraMetadata, 
HoodieWriteMetadata<HoodieData<WriteStatus>> result) {
     context.setJobStatus(this.getClass().getSimpleName(), "Commit write status 
collect: " + config.getTableName());
-    commit(extraMetadata, result, 
result.getWriteStatuses().map(WriteStatus::getStat).collectAsList());
-  }
-
-  protected void commit(Option<Map<String, String>> extraMetadata, 
HoodieWriteMetadata<HoodieData<WriteStatus>> result, List<HoodieWriteStat> 
writeStats) {
     String actionType = getCommitActionType();
     LOG.info("Committing " + instantTime + ", action Type " + actionType + ", 
operation Type " + operationType);
     result.setCommitted(true);
-    result.setWriteStats(writeStats);
+    if (!result.getWriteStats().isPresent()) {
+      
result.setWriteStats(result.getWriteStatuses().map(WriteStatus::getStat).collectAsList());
+    }
     // Finalize write
-    finalizeWrite(instantTime, writeStats, result);
+    finalizeWrite(instantTime, result.getWriteStats().get(), result);
     try {
       HoodieActiveTimeline activeTimeline = table.getActiveTimeline();
       HoodieCommitMetadata metadata = result.getCommitMetadata().get();