This is an automated email from the ASF dual-hosted git repository. pwason pushed a commit to branch release-0.14.0 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit d995bb8262cafa22253fa961557bbfcde6369dfb Author: Tim Brown <[email protected]> AuthorDate: Wed Aug 30 20:37:23 2023 -0500 [HUDI-6763] Optimize collect calls (#9561) --- .../table/action/commit/BaseSparkCommitActionExecutor.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java index 7383f428e0a..040cc798747 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java @@ -286,7 +286,9 @@ public abstract class BaseSparkCommitActionExecutor<T> extends @Override protected void setCommitMetadata(HoodieWriteMetadata<HoodieData<WriteStatus>> result) { - result.setCommitMetadata(Option.of(CommitUtils.buildMetadata(result.getWriteStatuses().map(WriteStatus::getStat).collectAsList(), + List<HoodieWriteStat> writeStats = result.getWriteStatuses().map(WriteStatus::getStat).collectAsList(); + result.setWriteStats(writeStats); + result.setCommitMetadata(Option.of(CommitUtils.buildMetadata(writeStats, result.getPartitionToReplaceFileIds(), extraMetadata, operationType, getSchemaToStoreInCommit(), getCommitActionType()))); } @@ -294,16 +296,14 @@ public abstract class BaseSparkCommitActionExecutor<T> extends @Override protected void commit(Option<Map<String, String>> extraMetadata, HoodieWriteMetadata<HoodieData<WriteStatus>> result) { context.setJobStatus(this.getClass().getSimpleName(), "Commit write status collect: " + config.getTableName()); - commit(extraMetadata, result, result.getWriteStatuses().map(WriteStatus::getStat).collectAsList()); - } - - protected void commit(Option<Map<String, String>> extraMetadata, HoodieWriteMetadata<HoodieData<WriteStatus>> result, List<HoodieWriteStat> writeStats) { String actionType = getCommitActionType(); LOG.info("Committing " + instantTime + ", action Type " + actionType + ", operation Type " + operationType); result.setCommitted(true); - result.setWriteStats(writeStats); + if (!result.getWriteStats().isPresent()) { + result.setWriteStats(result.getWriteStatuses().map(WriteStatus::getStat).collectAsList()); + } // Finalize write - finalizeWrite(instantTime, writeStats, result); + finalizeWrite(instantTime, result.getWriteStats().get(), result); try { HoodieActiveTimeline activeTimeline = table.getActiveTimeline(); HoodieCommitMetadata metadata = result.getCommitMetadata().get();
